You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by av...@apache.org on 2018/01/08 18:19:27 UTC
[01/28] ambari git commit: AMBARI-22740 : Fix integration test for
HBase in branch-3.0-ams due to UUID changes. (avijayan)
Repository: ambari
Updated Branches:
refs/heads/branch-3.0-ams a4293b7dc -> 42112e28d
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/source/RawMetricsSourceTest.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/source/RawMetricsSourceTest.java b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/source/RawMetricsSourceTest.java
deleted file mode 100644
index 254ee6c..0000000
--- a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/source/RawMetricsSourceTest.java
+++ /dev/null
@@ -1,141 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.source;
-
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.source.InternalSourceProvider.SOURCE_NAME.RAW_METRICS;
-import static org.easymock.EasyMock.capture;
-import static org.easymock.EasyMock.createNiceMock;
-import static org.easymock.EasyMock.expect;
-import static org.easymock.EasyMock.expectLastCall;
-import static org.easymock.EasyMock.replay;
-import static org.easymock.EasyMock.verify;
-import static org.powermock.api.easymock.PowerMock.mockStatic;
-import static org.powermock.api.easymock.PowerMock.replayAll;
-
-import java.util.Collection;
-import java.util.Collections;
-import java.util.TreeMap;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.metrics2.sink.timeline.TimelineMetric;
-import org.apache.hadoop.metrics2.sink.timeline.TimelineMetrics;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.sink.ExternalMetricsSink;
-import org.easymock.Capture;
-import org.junit.Before;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.powermock.core.classloader.annotations.PrepareForTest;
-import org.powermock.modules.junit4.PowerMockRunner;
-
-import junit.framework.Assert;
-
-@RunWith(PowerMockRunner.class)
-@PrepareForTest(TimelineMetricConfiguration.class)
-public class RawMetricsSourceTest {
-
- @Before
- public void setupConf() throws Exception {
- TimelineMetricConfiguration conf = new TimelineMetricConfiguration(new
- Configuration(), new Configuration());
- mockStatic(TimelineMetricConfiguration.class);
- expect(TimelineMetricConfiguration.getInstance()).andReturn(conf).anyTimes();
- replayAll();
- }
-
- @Test
- public void testRawMetricsSourcedAtFlushInterval() throws Exception {
- InternalSourceProvider internalSourceProvider = new DefaultInternalMetricsSourceProvider();
- ExternalMetricsSink rawMetricsSink = createNiceMock(ExternalMetricsSink.class);
- expect(rawMetricsSink.getFlushSeconds()).andReturn(1);
- expect(rawMetricsSink.getSinkTimeOutSeconds()).andReturn(1);
- Capture<Collection<TimelineMetrics>> metricsCapture = new Capture<>();
- rawMetricsSink.sinkMetricData(capture(metricsCapture));
- expectLastCall();
- replay(rawMetricsSink);
-
- InternalMetricsSource rawMetricsSource = internalSourceProvider.getInternalMetricsSource(RAW_METRICS, 1, rawMetricsSink);
- TimelineMetrics timelineMetrics = new TimelineMetrics();
-
- final long now = System.currentTimeMillis();
- TimelineMetric metric1 = new TimelineMetric();
- metric1.setMetricName("m1");
- metric1.setAppId("a1");
- metric1.setInstanceId("i1");
- metric1.setHostName("h1");
- metric1.setStartTime(now - 200);
- metric1.setMetricValues(new TreeMap<Long, Double>() {{
- put(now - 100, 1.0);
- put(now - 200, 2.0);
- }});
- timelineMetrics.getMetrics().add(metric1);
-
- rawMetricsSource.publishTimelineMetrics(Collections.singletonList(timelineMetrics));
-
- verify(rawMetricsSink);
- }
-
- @Test(timeout = 10000)
- public void testRawMetricsCachedAndSourced() throws Exception {
- ExternalMetricsSink rawMetricsSink = createNiceMock(ExternalMetricsSink.class);
- expect(rawMetricsSink.getFlushSeconds()).andReturn(2).anyTimes();
- expect(rawMetricsSink.getSinkTimeOutSeconds()).andReturn(1).anyTimes();
-
- class CaptureOnce<T> extends Capture<T> {
- @Override
- public void setValue(T value) {
- if (!hasCaptured()) {
- super.setValue(value);
- }
- }
- }
- Capture<Collection<TimelineMetrics>> metricsCapture = new CaptureOnce<>();
-
- rawMetricsSink.sinkMetricData(capture(metricsCapture));
- expectLastCall();
- replay(rawMetricsSink);
-
- InternalSourceProvider internalSourceProvider = new DefaultInternalMetricsSourceProvider();
- InternalMetricsSource rawMetricsSource = internalSourceProvider.getInternalMetricsSource(RAW_METRICS, 1, rawMetricsSink);
- TimelineMetrics timelineMetrics = new TimelineMetrics();
-
- final long now = System.currentTimeMillis();
- TimelineMetric metric1 = new TimelineMetric();
- metric1.setMetricName("m1");
- metric1.setAppId("a1");
- metric1.setInstanceId("i1");
- metric1.setHostName("h1");
- metric1.setStartTime(now - 200);
- metric1.setMetricValues(new TreeMap<Long, Double>() {{
- put(now - 100, 1.0);
- put(now - 200, 2.0);
- }});
- timelineMetrics.getMetrics().add(metric1);
-
- rawMetricsSource.publishTimelineMetrics(Collections.singletonList(timelineMetrics));
-
- // Wait on eviction
- Thread.sleep(5000);
-
- verify(rawMetricsSink);
-
- Assert.assertTrue(metricsCapture.hasCaptured());
- Assert.assertTrue(metricsCapture.getValue().iterator().next().getMetrics().iterator().next().equals(metric1));
- }
-
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/uuid/TimelineMetricUuidManagerTest.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/uuid/TimelineMetricUuidManagerTest.java b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/uuid/TimelineMetricUuidManagerTest.java
deleted file mode 100644
index d1b3f01..0000000
--- a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/uuid/TimelineMetricUuidManagerTest.java
+++ /dev/null
@@ -1,184 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.uuid;
-
-import static org.easymock.EasyMock.anyString;
-import static org.easymock.EasyMock.expect;
-import static org.easymock.EasyMock.replay;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.metrics2.sink.timeline.TimelineMetric;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.PhoenixHBaseAccessor;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.aggregators.TimelineClusterMetric;
-import org.easymock.EasyMock;
-import org.junit.Assert;
-import org.junit.Ignore;
-import org.junit.Test;
-
-import java.io.BufferedReader;
-import java.io.FileInputStream;
-import java.io.IOException;
-import java.io.InputStreamReader;
-import java.net.URL;
-import java.sql.SQLException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-public class TimelineMetricUuidManagerTest {
-
-
- private List<String> apps = Arrays.asList("namenode",
- "datanode", "master_hbase", "slave_hbase", "kafka_broker", "nimbus", "ams-hbase",
- "accumulo", "nodemanager", "resourcemanager", "ambari_server", "HOST", "timeline_metric_store_watcher",
- "jobhistoryserver", "hiveserver2", "hivemetastore", "applicationhistoryserver", "amssmoketestfake");
-
- private Map<String, Set<String>> metricSet = new HashMap<>(populateMetricWhitelistFromFile());
-
- @Test
- public void testHashBasedUuidForMetricName() throws SQLException {
-
- MetricUuidGenStrategy strategy = new HashBasedUuidGenStrategy();
- Map<String, TimelineClusterMetric> uuids = new HashMap<>();
- for (String app : metricSet.keySet()) {
- Set<String> metrics = metricSet.get(app);
- for (String metric : metrics) {
- TimelineClusterMetric timelineClusterMetric = new TimelineClusterMetric(metric, app, null, -1l);
- byte[] uuid = strategy.computeUuid(timelineClusterMetric, 16);
- Assert.assertNotNull(uuid);
- Assert.assertTrue(uuid.length == 16);
- String uuidStr = new String(uuid);
- Assert.assertFalse(uuids.containsKey(uuidStr) && !uuids.containsValue(timelineClusterMetric));
- if (uuids.containsKey(uuidStr) ) {
- if (!uuids.containsValue(timelineClusterMetric)) {
- System.out.println("COLLISION : " + timelineClusterMetric.toString() + " = " + uuids.get(uuidStr));
- }
- }
- uuids.put(uuidStr, timelineClusterMetric);
- }
- }
- }
-
- @Test
- public void testHaseBasedUuidForAppIds() throws SQLException {
-
- MetricUuidGenStrategy strategy = new HashBasedUuidGenStrategy();
- Map<String, TimelineClusterMetric> uuids = new HashMap<>();
- for (String app : metricSet.keySet()) {
- TimelineClusterMetric timelineClusterMetric = new TimelineClusterMetric("TestMetric", app, null, -1l);
- byte[] uuid = strategy.computeUuid(timelineClusterMetric, 16);
- String uuidStr = new String(uuid);
- if (uuids.containsKey(uuidStr) ) {
- if (!uuids.containsValue(timelineClusterMetric)) {
- System.out.println("COLLISION : " + timelineClusterMetric.toString() + " = " + uuids.get(uuidStr));
- }
- }
- uuids.put(uuidStr, timelineClusterMetric);
- }
- }
-
- @Test
- public void testHashBasedUuidForHostnames() throws SQLException {
-
- MetricUuidGenStrategy strategy = new HashBasedUuidGenStrategy();
- Map<String, String> uuids = new HashMap<>();
-
- List<String> hosts = new ArrayList<>();
- String hostPrefix = "TestHost.";
- String hostSuffix = ".ambari.apache.org";
-
- for (int i=0; i<=2000; i++) {
- hosts.add(hostPrefix + i + hostSuffix);
- }
-
- for (String host : hosts) {
- byte[] uuid = strategy.computeUuid(host, 4);
- Assert.assertNotNull(uuid);
- Assert.assertTrue(uuid.length == 4);
- String uuidStr = new String(uuid);
- Assert.assertFalse(uuids.containsKey(uuidStr));
- uuids.put(uuidStr, host);
- }
- }
-
-
- @Test
- public void testRandomUuidForWhitelistedMetrics() throws SQLException {
-
- MetricUuidGenStrategy strategy = new RandomUuidGenStrategy();
- Map<String, String> uuids = new HashMap<>();
- for (String app : metricSet.keySet()) {
- Set<String> metrics = metricSet.get(app);
- for (String metric : metrics) {
- byte[] uuid = strategy.computeUuid(new TimelineClusterMetric(metric, app, null, -1l), 16);
- Assert.assertNotNull(uuid);
- Assert.assertTrue(uuid.length == 16);
- String uuidStr = new String(uuid);
- Assert.assertFalse(uuids.containsKey(uuidStr) && !uuids.containsValue(metric));
- uuids.put(uuidStr, metric);
- }
- }
- }
-
- public Map<String, Set<String>> populateMetricWhitelistFromFile() {
-
-
- Map<String, Set<String>> metricSet = new HashMap<String, Set<String>>();
- FileInputStream fstream = null;
- BufferedReader br = null;
- String strLine;
- for (String appId : apps) {
- URL fileUrl = ClassLoader.getSystemResource("metrics_def/" + appId.toUpperCase() + ".dat");
-
- Set<String> metricsForApp = new HashSet<>();
- try {
- fstream = new FileInputStream(fileUrl.getPath());
- br = new BufferedReader(new InputStreamReader(fstream));
- while ((strLine = br.readLine()) != null) {
- strLine = strLine.trim();
- metricsForApp.add(strLine);
- }
- } catch (Exception ioEx) {
- System.out.println("Metrics for AppId " + appId + " not found.");
- } finally {
- if (br != null) {
- try {
- br.close();
- } catch (IOException e) {
- }
- }
-
- if (fstream != null) {
- try {
- fstream.close();
- } catch (IOException e) {
- }
- }
- }
- metricsForApp.add("live_hosts");
- metricSet.put(appId.contains("hbase") ? "hbase" : appId, metricsForApp);
- System.out.println("Found " + metricsForApp.size() + " metrics for appId = " + appId);
- }
- return metricSet;
- }
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/TestGenericObjectMapper.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/TestGenericObjectMapper.java b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/TestGenericObjectMapper.java
deleted file mode 100644
index d684a27..0000000
--- a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/TestGenericObjectMapper.java
+++ /dev/null
@@ -1,102 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.applicationhistoryservice.timeline;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.io.WritableComparator;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.GenericObjectMapper;
-import org.junit.Test;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import static org.junit.Assert.assertEquals;
-
-@InterfaceAudience.Private
-@InterfaceStability.Unstable
-public class TestGenericObjectMapper {
-
- @Test
- public void testEncoding() {
- testEncoding(Long.MAX_VALUE);
- testEncoding(Long.MIN_VALUE);
- testEncoding(0l);
- testEncoding(128l);
- testEncoding(256l);
- testEncoding(512l);
- testEncoding(-256l);
- }
-
- private static void testEncoding(long l) {
- byte[] b = GenericObjectMapper.writeReverseOrderedLong(l);
- assertEquals("error decoding", l,
- GenericObjectMapper.readReverseOrderedLong(b, 0));
- byte[] buf = new byte[16];
- System.arraycopy(b, 0, buf, 5, 8);
- assertEquals("error decoding at offset", l,
- GenericObjectMapper.readReverseOrderedLong(buf, 5));
- if (l > Long.MIN_VALUE) {
- byte[] a = GenericObjectMapper.writeReverseOrderedLong(l-1);
- assertEquals("error preserving ordering", 1,
- WritableComparator.compareBytes(a, 0, a.length, b, 0, b.length));
- }
- if (l < Long.MAX_VALUE) {
- byte[] c = GenericObjectMapper.writeReverseOrderedLong(l+1);
- assertEquals("error preserving ordering", 1,
- WritableComparator.compareBytes(b, 0, b.length, c, 0, c.length));
- }
- }
-
- private static void verify(Object o) throws IOException {
- assertEquals(o, GenericObjectMapper.read(GenericObjectMapper.write(o)));
- }
-
- @Test
- public void testValueTypes() throws IOException {
- verify(Integer.MAX_VALUE);
- verify(Integer.MIN_VALUE);
- assertEquals(Integer.MAX_VALUE, GenericObjectMapper.read(
- GenericObjectMapper.write((long) Integer.MAX_VALUE)));
- assertEquals(Integer.MIN_VALUE, GenericObjectMapper.read(
- GenericObjectMapper.write((long) Integer.MIN_VALUE)));
- verify((long)Integer.MAX_VALUE + 1l);
- verify((long)Integer.MIN_VALUE - 1l);
-
- verify(Long.MAX_VALUE);
- verify(Long.MIN_VALUE);
-
- assertEquals(42, GenericObjectMapper.read(GenericObjectMapper.write(42l)));
- verify(42);
- verify(1.23);
- verify("abc");
- verify(true);
- List<String> list = new ArrayList<String>();
- list.add("123");
- list.add("abc");
- verify(list);
- Map<String,String> map = new HashMap<String,String>();
- map.put("k1","v1");
- map.put("k2","v2");
- verify(map);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestTimelineWebServices.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestTimelineWebServices.java b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestTimelineWebServices.java
deleted file mode 100644
index 83e2a27..0000000
--- a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestTimelineWebServices.java
+++ /dev/null
@@ -1,124 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.server.applicationhistoryservice.webapp;
-
-import static org.junit.Assert.assertEquals;
-
-import javax.ws.rs.core.MediaType;
-
-import org.apache.hadoop.metrics2.sink.timeline.TimelineMetrics;
-import org.apache.hadoop.yarn.api.records.timeline.TimelineEntities;
-import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity;
-import org.apache.hadoop.yarn.api.records.timeline.TimelineEvent;
-import org.apache.hadoop.yarn.api.records.timeline.TimelineEvents;
-import org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TestTimelineMetricStore;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricStore;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.TimelineStore;
-import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
-import org.apache.hadoop.yarn.webapp.YarnJacksonJaxbJsonProvider;
-import org.junit.Test;
-
-import com.google.inject.Guice;
-import com.google.inject.Injector;
-import com.google.inject.servlet.GuiceServletContextListener;
-import com.google.inject.servlet.ServletModule;
-import com.sun.jersey.api.client.ClientResponse;
-import com.sun.jersey.api.client.WebResource;
-import com.sun.jersey.api.client.config.DefaultClientConfig;
-import com.sun.jersey.guice.spi.container.servlet.GuiceContainer;
-import com.sun.jersey.test.framework.JerseyTest;
-import com.sun.jersey.test.framework.WebAppDescriptor;
-
-import junit.framework.Assert;
-
-
-public class TestTimelineWebServices extends JerseyTest {
- private static TimelineMetricStore metricStore;
- private long beforeTime;
-
- private Injector injector = Guice.createInjector(new ServletModule() {
-
- @Override
- protected void configureServlets() {
- bind(YarnJacksonJaxbJsonProvider.class);
- bind(TimelineWebServices.class);
- bind(GenericExceptionHandler.class);
- try {
- metricStore = new TestTimelineMetricStore();
- } catch (Exception e) {
- Assert.fail();
- }
- bind(TimelineMetricStore.class).toInstance(metricStore);
- serve("/*").with(GuiceContainer.class);
- }
-
- });
-
- public class GuiceServletConfig extends GuiceServletContextListener {
-
- @Override
- protected Injector getInjector() {
- return injector;
- }
- }
-
- public TestTimelineWebServices() {
- super(new WebAppDescriptor.Builder(
- "org.apache.hadoop.yarn.server.applicationhistoryservice.webapp")
- .contextListenerClass(GuiceServletConfig.class)
- .filterClass(com.google.inject.servlet.GuiceFilter.class)
- .contextPath("jersey-guice-filter")
- .servletPath("/")
- .clientConfig(new DefaultClientConfig(YarnJacksonJaxbJsonProvider.class))
- .build());
- }
-
- @Test
- public void testAbout() throws Exception {
- WebResource r = resource();
- ClientResponse response = r.path("ws").path("v1").path("timeline")
- .accept(MediaType.APPLICATION_JSON)
- .get(ClientResponse.class);
- assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
- TimelineWebServices.AboutInfo about =
- response.getEntity(TimelineWebServices.AboutInfo.class);
- Assert.assertNotNull(about);
- Assert.assertEquals("AMS API", about.getAbout());
- }
-
- private static void verifyMetrics(TimelineMetrics metrics) {
- Assert.assertNotNull(metrics);
- Assert.assertEquals("cpu_user", metrics.getMetrics().get(0).getMetricName());
- Assert.assertEquals(3, metrics.getMetrics().get(0).getMetricValues().size());
- Assert.assertEquals("mem_free", metrics.getMetrics().get(1).getMetricName());
- Assert.assertEquals(3, metrics.getMetrics().get(1).getMetricValues().size());
- }
-
- @Test
- public void testGetMetrics() throws Exception {
- WebResource r = resource();
- ClientResponse response = r.path("ws").path("v1").path("timeline")
- .path("metrics").queryParam("metricNames", "cpu_user").queryParam("precision", "seconds")
- .accept(MediaType.APPLICATION_JSON)
- .get(ClientResponse.class);
- assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
- verifyMetrics(response.getEntity(TimelineMetrics.class));
- }
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/test/resources/loadsimulator/README
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/test/resources/loadsimulator/README b/ambari-metrics/ambari-metrics-timelineservice/src/test/resources/loadsimulator/README
index eec3e5d..c756a83 100644
--- a/ambari-metrics/ambari-metrics-timelineservice/src/test/resources/loadsimulator/README
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/test/resources/loadsimulator/README
@@ -41,7 +41,7 @@
2. Build the ambari-metrics-timelineservice jar.
3. Invoke the test using the command as follows.
- java -cp lib/*:ambari-metrics-timelineservice-<version>.jar org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.loadsimulator.jmetertest.jmetertest.AMSJMeterLoadTest -t <S/U>
+ java -cp lib/*:ambari-metrics-timelineservice-<version>.jar AMSJMeterLoadTest -t <S/U>
lib/* -> All the dependency JARs generated for the ambari-metrics-timelineservice JAR.
-t option => S-Sink simulator or U-UI simulator
[19/28] ambari git commit: AMBARI-22740 : Fix integration test for
HBase in branch-3.0-ams due to UUID changes. (avijayan)
Posted by av...@apache.org.
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/records/impl/pb/ApplicationFinishDataPBImpl.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/records/impl/pb/ApplicationFinishDataPBImpl.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/records/impl/pb/ApplicationFinishDataPBImpl.java
new file mode 100644
index 0000000..ba7e7f0
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/records/impl/pb/ApplicationFinishDataPBImpl.java
@@ -0,0 +1,226 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.metrics.records.impl.pb;
+
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
+import org.apache.hadoop.yarn.api.records.YarnApplicationState;
+import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationIdPBImpl;
+import org.apache.hadoop.yarn.api.records.impl.pb.ProtoUtils;
+import org.apache.hadoop.yarn.proto.ApplicationHistoryServerProtos.ApplicationFinishDataProto;
+import org.apache.hadoop.yarn.proto.ApplicationHistoryServerProtos.ApplicationFinishDataProtoOrBuilder;
+import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationIdProto;
+import org.apache.hadoop.yarn.proto.YarnProtos.FinalApplicationStatusProto;
+import org.apache.hadoop.yarn.proto.YarnProtos.YarnApplicationStateProto;
+import org.apache.ambari.metrics.records.ApplicationFinishData;
+
+import com.google.protobuf.TextFormat;
+
+public class ApplicationFinishDataPBImpl extends ApplicationFinishData {
+
+ ApplicationFinishDataProto proto = ApplicationFinishDataProto
+ .getDefaultInstance();
+ ApplicationFinishDataProto.Builder builder = null;
+ boolean viaProto = false;
+
+ private ApplicationId applicationId;
+
+ public ApplicationFinishDataPBImpl() {
+ builder = ApplicationFinishDataProto.newBuilder();
+ }
+
+ public ApplicationFinishDataPBImpl(ApplicationFinishDataProto proto) {
+ this.proto = proto;
+ viaProto = true;
+ }
+
+ @Override
+ public ApplicationId getApplicationId() {
+ if (this.applicationId != null) {
+ return this.applicationId;
+ }
+ ApplicationFinishDataProtoOrBuilder p = viaProto ? proto : builder;
+ if (!p.hasApplicationId()) {
+ return null;
+ }
+ this.applicationId = convertFromProtoFormat(p.getApplicationId());
+ return this.applicationId;
+ }
+
+ @Override
+ public void setApplicationId(ApplicationId applicationId) {
+ maybeInitBuilder();
+ if (applicationId == null) {
+ builder.clearApplicationId();
+ }
+ this.applicationId = applicationId;
+ }
+
+ @Override
+ public long getFinishTime() {
+ ApplicationFinishDataProtoOrBuilder p = viaProto ? proto : builder;
+ return p.getFinishTime();
+ }
+
+ @Override
+ public void setFinishTime(long finishTime) {
+ maybeInitBuilder();
+ builder.setFinishTime(finishTime);
+ }
+
+ @Override
+ public String getDiagnosticsInfo() {
+ ApplicationFinishDataProtoOrBuilder p = viaProto ? proto : builder;
+ if (!p.hasDiagnosticsInfo()) {
+ return null;
+ }
+ return p.getDiagnosticsInfo();
+ }
+
+ @Override
+ public void setDiagnosticsInfo(String diagnosticsInfo) {
+ maybeInitBuilder();
+ if (diagnosticsInfo == null) {
+ builder.clearDiagnosticsInfo();
+ return;
+ }
+ builder.setDiagnosticsInfo(diagnosticsInfo);
+ }
+
+ @Override
+ public FinalApplicationStatus getFinalApplicationStatus() {
+ ApplicationFinishDataProtoOrBuilder p = viaProto ? proto : builder;
+ if (!p.hasFinalApplicationStatus()) {
+ return null;
+ }
+ return convertFromProtoFormat(p.getFinalApplicationStatus());
+ }
+
+ @Override
+ public void setFinalApplicationStatus(
+ FinalApplicationStatus finalApplicationStatus) {
+ maybeInitBuilder();
+ if (finalApplicationStatus == null) {
+ builder.clearFinalApplicationStatus();
+ return;
+ }
+ builder
+ .setFinalApplicationStatus(convertToProtoFormat(finalApplicationStatus));
+ }
+
+ @Override
+ public YarnApplicationState getYarnApplicationState() {
+ ApplicationFinishDataProtoOrBuilder p = viaProto ? proto : builder;
+ if (!p.hasYarnApplicationState()) {
+ return null;
+ }
+ return convertFromProtoFormat(p.getYarnApplicationState());
+ }
+
+ @Override
+ public void setYarnApplicationState(YarnApplicationState state) {
+ maybeInitBuilder();
+ if (state == null) {
+ builder.clearYarnApplicationState();
+ return;
+ }
+ builder.setYarnApplicationState(convertToProtoFormat(state));
+ }
+
+ public ApplicationFinishDataProto getProto() {
+ mergeLocalToProto();
+ proto = viaProto ? proto : builder.build();
+ viaProto = true;
+ return proto;
+ }
+
+ @Override
+ public int hashCode() {
+ return getProto().hashCode();
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (other == null)
+ return false;
+ if (other.getClass().isAssignableFrom(this.getClass())) {
+ return this.getProto().equals(this.getClass().cast(other).getProto());
+ }
+ return false;
+ }
+
+ @Override
+ public String toString() {
+ return TextFormat.shortDebugString(getProto());
+ }
+
+ private void mergeLocalToBuilder() {
+ if (this.applicationId != null
+ && !((ApplicationIdPBImpl) this.applicationId).getProto().equals(
+ builder.getApplicationId())) {
+ builder.setApplicationId(convertToProtoFormat(this.applicationId));
+ }
+ }
+
+ private void mergeLocalToProto() {
+ if (viaProto) {
+ maybeInitBuilder();
+ }
+ mergeLocalToBuilder();
+ proto = builder.build();
+ viaProto = true;
+ }
+
+ private void maybeInitBuilder() {
+ if (viaProto || builder == null) {
+ builder = ApplicationFinishDataProto.newBuilder(proto);
+ }
+ viaProto = false;
+ }
+
+ private ApplicationIdProto convertToProtoFormat(ApplicationId applicationId) {
+ return ((ApplicationIdPBImpl) applicationId).getProto();
+ }
+
+ private ApplicationIdPBImpl convertFromProtoFormat(
+ ApplicationIdProto applicationId) {
+ return new ApplicationIdPBImpl(applicationId);
+ }
+
+ private FinalApplicationStatus convertFromProtoFormat(
+ FinalApplicationStatusProto finalApplicationStatus) {
+ return ProtoUtils.convertFromProtoFormat(finalApplicationStatus);
+ }
+
+ private FinalApplicationStatusProto convertToProtoFormat(
+ FinalApplicationStatus finalApplicationStatus) {
+ return ProtoUtils.convertToProtoFormat(finalApplicationStatus);
+ }
+
+ private YarnApplicationStateProto convertToProtoFormat(
+ YarnApplicationState state) {
+ return ProtoUtils.convertToProtoFormat(state);
+ }
+
+ private YarnApplicationState convertFromProtoFormat(
+ YarnApplicationStateProto yarnApplicationState) {
+ return ProtoUtils.convertFromProtoFormat(yarnApplicationState);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/records/impl/pb/ApplicationStartDataPBImpl.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/records/impl/pb/ApplicationStartDataPBImpl.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/records/impl/pb/ApplicationStartDataPBImpl.java
new file mode 100644
index 0000000..f47ab00
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/records/impl/pb/ApplicationStartDataPBImpl.java
@@ -0,0 +1,229 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.metrics.records.impl.pb;
+
+import org.apache.ambari.metrics.records.ApplicationStartData;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationIdPBImpl;
+import org.apache.hadoop.yarn.proto.ApplicationHistoryServerProtos.ApplicationStartDataProto;
+import org.apache.hadoop.yarn.proto.ApplicationHistoryServerProtos.ApplicationStartDataProtoOrBuilder;
+import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationIdProto;
+
+import com.google.protobuf.TextFormat;
+
+public class ApplicationStartDataPBImpl extends ApplicationStartData {
+
+ ApplicationStartDataProto proto = ApplicationStartDataProto
+ .getDefaultInstance();
+ ApplicationStartDataProto.Builder builder = null;
+ boolean viaProto = false;
+
+ private ApplicationId applicationId;
+
+ public ApplicationStartDataPBImpl() {
+ builder = ApplicationStartDataProto.newBuilder();
+ }
+
+ public ApplicationStartDataPBImpl(ApplicationStartDataProto proto) {
+ this.proto = proto;
+ viaProto = true;
+ }
+
+ @Override
+ public ApplicationId getApplicationId() {
+ if (this.applicationId != null) {
+ return this.applicationId;
+ }
+ ApplicationStartDataProtoOrBuilder p = viaProto ? proto : builder;
+ if (!p.hasApplicationId()) {
+ return null;
+ }
+ this.applicationId = convertFromProtoFormat(p.getApplicationId());
+ return this.applicationId;
+ }
+
+ @Override
+ public void setApplicationId(ApplicationId applicationId) {
+ maybeInitBuilder();
+ if (applicationId == null) {
+ builder.clearApplicationId();
+ }
+ this.applicationId = applicationId;
+ }
+
+ @Override
+ public String getApplicationName() {
+ ApplicationStartDataProtoOrBuilder p = viaProto ? proto : builder;
+ if (!p.hasApplicationName()) {
+ return null;
+ }
+ return p.getApplicationName();
+ }
+
+ @Override
+ public void setApplicationName(String applicationName) {
+ maybeInitBuilder();
+ if (applicationName == null) {
+ builder.clearApplicationName();
+ return;
+ }
+ builder.setApplicationName(applicationName);
+ }
+
+ @Override
+ public String getApplicationType() {
+ ApplicationStartDataProtoOrBuilder p = viaProto ? proto : builder;
+ if (!p.hasApplicationType()) {
+ return null;
+ }
+ return p.getApplicationType();
+ }
+
+ @Override
+ public void setApplicationType(String applicationType) {
+ maybeInitBuilder();
+ if (applicationType == null) {
+ builder.clearApplicationType();
+ return;
+ }
+ builder.setApplicationType(applicationType);
+ }
+
+ @Override
+ public String getUser() {
+ ApplicationStartDataProtoOrBuilder p = viaProto ? proto : builder;
+ if (!p.hasUser()) {
+ return null;
+ }
+ return p.getUser();
+ }
+
+ @Override
+ public void setUser(String user) {
+ maybeInitBuilder();
+ if (user == null) {
+ builder.clearUser();
+ return;
+ }
+ builder.setUser(user);
+ }
+
+ @Override
+ public String getQueue() {
+ ApplicationStartDataProtoOrBuilder p = viaProto ? proto : builder;
+ if (!p.hasQueue()) {
+ return null;
+ }
+ return p.getQueue();
+ }
+
+ @Override
+ public void setQueue(String queue) {
+ maybeInitBuilder();
+ if (queue == null) {
+ builder.clearQueue();
+ return;
+ }
+ builder.setQueue(queue);
+ }
+
+ @Override
+ public long getSubmitTime() {
+ ApplicationStartDataProtoOrBuilder p = viaProto ? proto : builder;
+ return p.getSubmitTime();
+ }
+
+ @Override
+ public void setSubmitTime(long submitTime) {
+ maybeInitBuilder();
+ builder.setSubmitTime(submitTime);
+ }
+
+ @Override
+ public long getStartTime() {
+ ApplicationStartDataProtoOrBuilder p = viaProto ? proto : builder;
+ return p.getStartTime();
+ }
+
+ @Override
+ public void setStartTime(long startTime) {
+ maybeInitBuilder();
+ builder.setStartTime(startTime);
+ }
+
+ public ApplicationStartDataProto getProto() {
+ mergeLocalToProto();
+ proto = viaProto ? proto : builder.build();
+ viaProto = true;
+ return proto;
+ }
+
+ @Override
+ public int hashCode() {
+ return getProto().hashCode();
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (other == null)
+ return false;
+ if (other.getClass().isAssignableFrom(this.getClass())) {
+ return this.getProto().equals(this.getClass().cast(other).getProto());
+ }
+ return false;
+ }
+
+ @Override
+ public String toString() {
+ return TextFormat.shortDebugString(getProto());
+ }
+
+ private void mergeLocalToBuilder() {
+ if (this.applicationId != null
+ && !((ApplicationIdPBImpl) this.applicationId).getProto().equals(
+ builder.getApplicationId())) {
+ builder.setApplicationId(convertToProtoFormat(this.applicationId));
+ }
+ }
+
+ private void mergeLocalToProto() {
+ if (viaProto) {
+ maybeInitBuilder();
+ }
+ mergeLocalToBuilder();
+ proto = builder.build();
+ viaProto = true;
+ }
+
+ private void maybeInitBuilder() {
+ if (viaProto || builder == null) {
+ builder = ApplicationStartDataProto.newBuilder(proto);
+ }
+ viaProto = false;
+ }
+
+ private ApplicationIdProto convertToProtoFormat(ApplicationId applicationId) {
+ return ((ApplicationIdPBImpl) applicationId).getProto();
+ }
+
+ private ApplicationIdPBImpl convertFromProtoFormat(
+ ApplicationIdProto applicationId) {
+ return new ApplicationIdPBImpl(applicationId);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/records/impl/pb/ContainerFinishDataPBImpl.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/records/impl/pb/ContainerFinishDataPBImpl.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/records/impl/pb/ContainerFinishDataPBImpl.java
new file mode 100644
index 0000000..e9aeb92
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/records/impl/pb/ContainerFinishDataPBImpl.java
@@ -0,0 +1,204 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.metrics.records.impl.pb;
+
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.ContainerState;
+import org.apache.hadoop.yarn.api.records.impl.pb.ContainerIdPBImpl;
+import org.apache.hadoop.yarn.api.records.impl.pb.ProtoUtils;
+import org.apache.hadoop.yarn.proto.ApplicationHistoryServerProtos.ContainerFinishDataProto;
+import org.apache.hadoop.yarn.proto.ApplicationHistoryServerProtos.ContainerFinishDataProtoOrBuilder;
+import org.apache.hadoop.yarn.proto.YarnProtos.ContainerIdProto;
+import org.apache.hadoop.yarn.proto.YarnProtos.ContainerStateProto;
+import org.apache.ambari.metrics.records.ContainerFinishData;
+
+import com.google.protobuf.TextFormat;
+
+public class ContainerFinishDataPBImpl extends ContainerFinishData {
+
+ ContainerFinishDataProto proto = ContainerFinishDataProto
+ .getDefaultInstance();
+ ContainerFinishDataProto.Builder builder = null;
+ boolean viaProto = false;
+
+ private ContainerId containerId;
+
+ public ContainerFinishDataPBImpl() {
+ builder = ContainerFinishDataProto.newBuilder();
+ }
+
+ public ContainerFinishDataPBImpl(ContainerFinishDataProto proto) {
+ this.proto = proto;
+ viaProto = true;
+ }
+
+ @Override
+ public ContainerId getContainerId() {
+ if (this.containerId != null) {
+ return this.containerId;
+ }
+ ContainerFinishDataProtoOrBuilder p = viaProto ? proto : builder;
+ if (!p.hasContainerId()) {
+ return null;
+ }
+ this.containerId = convertFromProtoFormat(p.getContainerId());
+ return this.containerId;
+ }
+
+ @Override
+ public void setContainerId(ContainerId containerId) {
+ maybeInitBuilder();
+ if (containerId == null) {
+ builder.clearContainerId();
+ }
+ this.containerId = containerId;
+ }
+
+ @Override
+ public long getFinishTime() {
+ ContainerFinishDataProtoOrBuilder p = viaProto ? proto : builder;
+ return p.getFinishTime();
+ }
+
+ @Override
+ public void setFinishTime(long finishTime) {
+ maybeInitBuilder();
+ builder.setFinishTime(finishTime);
+ }
+
+ @Override
+ public String getDiagnosticsInfo() {
+ ContainerFinishDataProtoOrBuilder p = viaProto ? proto : builder;
+ if (!p.hasDiagnosticsInfo()) {
+ return null;
+ }
+ return p.getDiagnosticsInfo();
+ }
+
+ @Override
+ public void setDiagnosticsInfo(String diagnosticsInfo) {
+ maybeInitBuilder();
+ if (diagnosticsInfo == null) {
+ builder.clearDiagnosticsInfo();
+ return;
+ }
+ builder.setDiagnosticsInfo(diagnosticsInfo);
+ }
+
+ @Override
+ public int getContainerExitStatus() {
+ ContainerFinishDataProtoOrBuilder p = viaProto ? proto : builder;
+ return p.getContainerExitStatus();
+ }
+
+ @Override
+ public ContainerState getContainerState() {
+ ContainerFinishDataProtoOrBuilder p = viaProto ? proto : builder;
+ if (!p.hasContainerState()) {
+ return null;
+ }
+ return convertFromProtoFormat(p.getContainerState());
+ }
+
+ @Override
+ public void setContainerState(ContainerState state) {
+ maybeInitBuilder();
+ if (state == null) {
+ builder.clearContainerState();
+ return;
+ }
+ builder.setContainerState(convertToProtoFormat(state));
+ }
+
+ @Override
+ public void setContainerExitStatus(int containerExitStatus) {
+ maybeInitBuilder();
+ builder.setContainerExitStatus(containerExitStatus);
+ }
+
+ public ContainerFinishDataProto getProto() {
+ mergeLocalToProto();
+ proto = viaProto ? proto : builder.build();
+ viaProto = true;
+ return proto;
+ }
+
+ @Override
+ public int hashCode() {
+ return getProto().hashCode();
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (other == null)
+ return false;
+ if (other.getClass().isAssignableFrom(this.getClass())) {
+ return this.getProto().equals(this.getClass().cast(other).getProto());
+ }
+ return false;
+ }
+
+ @Override
+ public String toString() {
+ return TextFormat.shortDebugString(getProto());
+ }
+
+ private void mergeLocalToBuilder() {
+ if (this.containerId != null
+ && !((ContainerIdPBImpl) this.containerId).getProto().equals(
+ builder.getContainerId())) {
+ builder.setContainerId(convertToProtoFormat(this.containerId));
+ }
+ }
+
+ private void mergeLocalToProto() {
+ if (viaProto) {
+ maybeInitBuilder();
+ }
+ mergeLocalToBuilder();
+ proto = builder.build();
+ viaProto = true;
+ }
+
+ private void maybeInitBuilder() {
+ if (viaProto || builder == null) {
+ builder = ContainerFinishDataProto.newBuilder(proto);
+ }
+ viaProto = false;
+ }
+
+ private ContainerIdProto convertToProtoFormat(ContainerId containerId) {
+ return ((ContainerIdPBImpl) containerId).getProto();
+ }
+
+ private ContainerIdPBImpl
+ convertFromProtoFormat(ContainerIdProto containerId) {
+ return new ContainerIdPBImpl(containerId);
+ }
+
+ private ContainerStateProto convertToProtoFormat(ContainerState state) {
+ return ProtoUtils.convertToProtoFormat(state);
+ }
+
+ private ContainerState convertFromProtoFormat(
+ ContainerStateProto containerState) {
+ return ProtoUtils.convertFromProtoFormat(containerState);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/records/impl/pb/ContainerStartDataPBImpl.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/records/impl/pb/ContainerStartDataPBImpl.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/records/impl/pb/ContainerStartDataPBImpl.java
new file mode 100644
index 0000000..e7d0762
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/records/impl/pb/ContainerStartDataPBImpl.java
@@ -0,0 +1,258 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.metrics.records.impl.pb;
+
+import org.apache.ambari.metrics.records.ContainerStartData;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.api.records.Priority;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.impl.pb.ContainerIdPBImpl;
+import org.apache.hadoop.yarn.api.records.impl.pb.NodeIdPBImpl;
+import org.apache.hadoop.yarn.api.records.impl.pb.PriorityPBImpl;
+import org.apache.hadoop.yarn.api.records.impl.pb.ResourcePBImpl;
+import org.apache.hadoop.yarn.proto.ApplicationHistoryServerProtos.ContainerStartDataProto;
+import org.apache.hadoop.yarn.proto.ApplicationHistoryServerProtos.ContainerStartDataProtoOrBuilder;
+import org.apache.hadoop.yarn.proto.YarnProtos.ContainerIdProto;
+import org.apache.hadoop.yarn.proto.YarnProtos.NodeIdProto;
+import org.apache.hadoop.yarn.proto.YarnProtos.PriorityProto;
+import org.apache.hadoop.yarn.proto.YarnProtos.ResourceProto;
+
+import com.google.protobuf.TextFormat;
+
+public class ContainerStartDataPBImpl extends ContainerStartData {
+
+ ContainerStartDataProto proto = ContainerStartDataProto.getDefaultInstance();
+ ContainerStartDataProto.Builder builder = null;
+ boolean viaProto = false;
+
+ private ContainerId containerId;
+ private Resource resource;
+ private NodeId nodeId;
+ private Priority priority;
+
+ public ContainerStartDataPBImpl() {
+ builder = ContainerStartDataProto.newBuilder();
+ }
+
+ public ContainerStartDataPBImpl(ContainerStartDataProto proto) {
+ this.proto = proto;
+ viaProto = true;
+ }
+
+ @Override
+ public ContainerId getContainerId() {
+ if (this.containerId != null) {
+ return this.containerId;
+ }
+ ContainerStartDataProtoOrBuilder p = viaProto ? proto : builder;
+ if (!p.hasContainerId()) {
+ return null;
+ }
+ this.containerId = convertFromProtoFormat(p.getContainerId());
+ return this.containerId;
+ }
+
+ @Override
+ public void setContainerId(ContainerId containerId) {
+ maybeInitBuilder();
+ if (containerId == null) {
+ builder.clearContainerId();
+ }
+ this.containerId = containerId;
+ }
+
+ @Override
+ public Resource getAllocatedResource() {
+ if (this.resource != null) {
+ return this.resource;
+ }
+ ContainerStartDataProtoOrBuilder p = viaProto ? proto : builder;
+ if (!p.hasAllocatedResource()) {
+ return null;
+ }
+ this.resource = convertFromProtoFormat(p.getAllocatedResource());
+ return this.resource;
+ }
+
+ @Override
+ public void setAllocatedResource(Resource resource) {
+ maybeInitBuilder();
+ if (resource == null) {
+ builder.clearAllocatedResource();
+ }
+ this.resource = resource;
+ }
+
+ @Override
+ public NodeId getAssignedNode() {
+ if (this.nodeId != null) {
+ return this.nodeId;
+ }
+ ContainerStartDataProtoOrBuilder p = viaProto ? proto : builder;
+ if (!p.hasAssignedNodeId()) {
+ return null;
+ }
+ this.nodeId = convertFromProtoFormat(p.getAssignedNodeId());
+ return this.nodeId;
+ }
+
+ @Override
+ public void setAssignedNode(NodeId nodeId) {
+ maybeInitBuilder();
+ if (nodeId == null) {
+ builder.clearAssignedNodeId();
+ }
+ this.nodeId = nodeId;
+ }
+
+ @Override
+ public Priority getPriority() {
+ if (this.priority != null) {
+ return this.priority;
+ }
+ ContainerStartDataProtoOrBuilder p = viaProto ? proto : builder;
+ if (!p.hasPriority()) {
+ return null;
+ }
+ this.priority = convertFromProtoFormat(p.getPriority());
+ return this.priority;
+ }
+
+ @Override
+ public void setPriority(Priority priority) {
+ maybeInitBuilder();
+ if (priority == null) {
+ builder.clearPriority();
+ }
+ this.priority = priority;
+ }
+
+ @Override
+ public long getStartTime() {
+ ContainerStartDataProtoOrBuilder p = viaProto ? proto : builder;
+ return p.getStartTime();
+ }
+
+ @Override
+ public void setStartTime(long startTime) {
+ maybeInitBuilder();
+ builder.setStartTime(startTime);
+ }
+
+ public ContainerStartDataProto getProto() {
+ mergeLocalToProto();
+ proto = viaProto ? proto : builder.build();
+ viaProto = true;
+ return proto;
+ }
+
+ @Override
+ public int hashCode() {
+ return getProto().hashCode();
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (other == null)
+ return false;
+ if (other.getClass().isAssignableFrom(this.getClass())) {
+ return this.getProto().equals(this.getClass().cast(other).getProto());
+ }
+ return false;
+ }
+
+ @Override
+ public String toString() {
+ return TextFormat.shortDebugString(getProto());
+ }
+
+ private void mergeLocalToBuilder() {
+ if (this.containerId != null
+ && !((ContainerIdPBImpl) this.containerId).getProto().equals(
+ builder.getContainerId())) {
+ builder.setContainerId(convertToProtoFormat(this.containerId));
+ }
+ if (this.resource != null
+ && !((ResourcePBImpl) this.resource).getProto().equals(
+ builder.getAllocatedResource())) {
+ builder.setAllocatedResource(convertToProtoFormat(this.resource));
+ }
+ if (this.nodeId != null
+ && !((NodeIdPBImpl) this.nodeId).getProto().equals(
+ builder.getAssignedNodeId())) {
+ builder.setAssignedNodeId(convertToProtoFormat(this.nodeId));
+ }
+ if (this.priority != null
+ && !((PriorityPBImpl) this.priority).getProto().equals(
+ builder.getPriority())) {
+ builder.setPriority(convertToProtoFormat(this.priority));
+ }
+ }
+
+ private void mergeLocalToProto() {
+ if (viaProto) {
+ maybeInitBuilder();
+ }
+ mergeLocalToBuilder();
+ proto = builder.build();
+ viaProto = true;
+ }
+
+ private void maybeInitBuilder() {
+ if (viaProto || builder == null) {
+ builder = ContainerStartDataProto.newBuilder(proto);
+ }
+ viaProto = false;
+ }
+
+ private ContainerIdProto convertToProtoFormat(ContainerId containerId) {
+ return ((ContainerIdPBImpl) containerId).getProto();
+ }
+
+ private ContainerIdPBImpl
+ convertFromProtoFormat(ContainerIdProto containerId) {
+ return new ContainerIdPBImpl(containerId);
+ }
+
+ private ResourceProto convertToProtoFormat(Resource resource) {
+ return ((ResourcePBImpl) resource).getProto();
+ }
+
+ private ResourcePBImpl convertFromProtoFormat(ResourceProto resource) {
+ return new ResourcePBImpl(resource);
+ }
+
+ private NodeIdProto convertToProtoFormat(NodeId nodeId) {
+ return ((NodeIdPBImpl) nodeId).getProto();
+ }
+
+ private NodeIdPBImpl convertFromProtoFormat(NodeIdProto nodeId) {
+ return new NodeIdPBImpl(nodeId);
+ }
+
+ private PriorityProto convertToProtoFormat(Priority priority) {
+ return ((PriorityPBImpl) priority).getProto();
+ }
+
+ private PriorityPBImpl convertFromProtoFormat(PriorityProto priority) {
+ return new PriorityPBImpl(priority);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/timeline/GenericObjectMapper.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/timeline/GenericObjectMapper.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/timeline/GenericObjectMapper.java
new file mode 100644
index 0000000..b802a88
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/timeline/GenericObjectMapper.java
@@ -0,0 +1,135 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.metrics.timeline;
+
+import java.io.IOException;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.codehaus.jackson.map.ObjectMapper;
+import org.codehaus.jackson.map.ObjectReader;
+import org.codehaus.jackson.map.ObjectWriter;
+
+/**
+ * A utility class providing methods for serializing and deserializing
+ * objects. The {@link #write(Object)} and {@link #read(byte[])} methods are
+ * used by the {@link LeveldbTimelineStore} to store and retrieve arbitrary
+ * JSON, while the {@link #writeReverseOrderedLong} and {@link
+ * #readReverseOrderedLong} methods are used to sort entities in descending
+ * start time order.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+public class GenericObjectMapper {
+ private static final byte[] EMPTY_BYTES = new byte[0];
+
+ public static final ObjectReader OBJECT_READER;
+ public static final ObjectWriter OBJECT_WRITER;
+
+ static {
+ ObjectMapper mapper = new ObjectMapper();
+ OBJECT_READER = mapper.reader(Object.class);
+ OBJECT_WRITER = mapper.writer();
+ }
+
+ /**
+ * Serializes an Object into a byte array. Along with {@link #read(byte[])},
+ * can be used to serialize an Object and deserialize it into an Object of
+ * the same type without needing to specify the Object's type,
+ * as long as it is one of the JSON-compatible objects understood by
+ * ObjectMapper.
+ *
+ * @param o An Object
+ * @return A byte array representation of the Object
+ * @throws IOException if there is a write error
+ */
+ public static byte[] write(Object o) throws IOException {
+ if (o == null) {
+ return EMPTY_BYTES;
+ }
+ return OBJECT_WRITER.writeValueAsBytes(o);
+ }
+
+ /**
+ * Deserializes an Object from a byte array created with
+ * {@link #write(Object)}.
+ *
+ * @param b A byte array
+ * @return An Object
+ * @throws IOException if there is a read error
+ */
+ public static Object read(byte[] b) throws IOException {
+ return read(b, 0);
+ }
+
+ /**
+ * Deserializes an Object from a byte array at a specified offset, assuming
+ * the bytes were created with {@link #write(Object)}.
+ *
+ * @param b A byte array
+ * @param offset Offset into the array
+ * @return An Object
+ * @throws IOException if there is a read error
+ */
+ public static Object read(byte[] b, int offset) throws IOException {
+ if (b == null || b.length == 0) {
+ return null;
+ }
+ return OBJECT_READER.readValue(b, offset, b.length - offset);
+ }
+
+ /**
+ * Converts a long to a 8-byte array so that lexicographic ordering of the
+ * produced byte arrays sort the longs in descending order.
+ *
+ * @param l A long
+ * @return A byte array
+ */
+ public static byte[] writeReverseOrderedLong(long l) {
+ byte[] b = new byte[8];
+ return writeReverseOrderedLong(l, b, 0);
+ }
+
+ public static byte[] writeReverseOrderedLong(long l, byte[] b, int offset) {
+ b[offset] = (byte)(0x7f ^ ((l >> 56) & 0xff));
+ for (int i = offset+1; i < offset+7; i++) {
+ b[i] = (byte)(0xff ^ ((l >> 8*(7-i)) & 0xff));
+ }
+ b[offset+7] = (byte)(0xff ^ (l & 0xff));
+ return b;
+ }
+
+ /**
+ * Reads 8 bytes from an array starting at the specified offset and
+ * converts them to a long. The bytes are assumed to have been created
+ * with {@link #writeReverseOrderedLong}.
+ *
+ * @param b A byte array
+ * @param offset An offset into the byte array
+ * @return A long
+ */
+ public static long readReverseOrderedLong(byte[] b, int offset) {
+ long l = b[offset] & 0xff;
+ for (int i = 1; i < 8; i++) {
+ l = l << 8;
+ l = l | (b[offset+i]&0xff);
+ }
+ return l ^ 0x7fffffffffffffffl;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/timeline/NameValuePair.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/timeline/NameValuePair.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/timeline/NameValuePair.java
new file mode 100644
index 0000000..d31a658
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/timeline/NameValuePair.java
@@ -0,0 +1,59 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.metrics.timeline;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+/**
+ * A class holding a name and value pair, used for specifying filters in
+ * {@link TimelineReader}.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+public class NameValuePair {
+ String name;
+ Object value;
+
+ public NameValuePair(String name, Object value) {
+ this.name = name;
+ this.value = value;
+ }
+
+ /**
+ * Get the name.
+ * @return The name.
+ */
+ public String getName() {
+
+ return name;
+ }
+
+ /**
+ * Get the value.
+ * @return The value.
+ */
+ public Object getValue() {
+ return value;
+ }
+
+ @Override
+ public String toString() {
+ return "{ name: " + name + ", value: " + value + " }";
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/timeline/TimelineReader.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/timeline/TimelineReader.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/timeline/TimelineReader.java
new file mode 100644
index 0000000..7c6efdb
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/timeline/TimelineReader.java
@@ -0,0 +1,155 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.metrics.timeline;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.EnumSet;
+import java.util.Set;
+import java.util.SortedSet;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.yarn.api.records.timeline.TimelineEntities;
+import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity;
+import org.apache.hadoop.yarn.api.records.timeline.TimelineEvents;
+
+/**
+ * This interface is for retrieving timeline information.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+public interface TimelineReader {
+
+ /**
+ * Possible fields to retrieve for {@link #getEntities} and {@link #getEntity}
+ * .
+ */
+ enum Field {
+ EVENTS,
+ RELATED_ENTITIES,
+ PRIMARY_FILTERS,
+ OTHER_INFO,
+ LAST_EVENT_ONLY
+ }
+
+ /**
+ * Default limit for {@link #getEntities} and {@link #getEntityTimelines}.
+ */
+ final long DEFAULT_LIMIT = 100;
+
+ /**
+ * This method retrieves a list of entity information, {@link TimelineEntity},
+ * sorted by the starting timestamp for the entity, descending. The starting
+ * timestamp of an entity is a timestamp specified by the client. If it is not
+ * explicitly specified, it will be chosen by the store to be the earliest
+ * timestamp of the events received in the first put for the entity.
+ *
+ * @param entityType
+ * The type of entities to return (required).
+ * @param limit
+ * A limit on the number of entities to return. If null, defaults to
+ * {@link #DEFAULT_LIMIT}.
+ * @param windowStart
+ * The earliest start timestamp to retrieve (exclusive). If null,
+ * defaults to retrieving all entities until the limit is reached.
+ * @param windowEnd
+ * The latest start timestamp to retrieve (inclusive). If null,
+ * defaults to {@link Long#MAX_VALUE}
+ * @param fromId
+ * If fromId is not null, retrieve entities earlier than and
+ * including the specified ID. If no start time is found for the
+ * specified ID, an empty list of entities will be returned. The
+ * windowEnd parameter will take precedence if the start time of this
+ * entity falls later than windowEnd.
+ * @param fromTs
+ * If fromTs is not null, ignore entities that were inserted into the
+ * store after the given timestamp. The entity's insert timestamp
+ * used for this comparison is the store's system time when the first
+ * put for the entity was received (not the entity's start time).
+ * @param primaryFilter
+ * Retrieves only entities that have the specified primary filter. If
+ * null, retrieves all entities. This is an indexed retrieval, and no
+ * entities that do not match the filter are scanned.
+ * @param secondaryFilters
+ * Retrieves only entities that have exact matches for all the
+ * specified filters in their primary filters or other info. This is
+ * not an indexed retrieval, so all entities are scanned but only
+ * those matching the filters are returned.
+ * @param fieldsToRetrieve
+ * Specifies which fields of the entity object to retrieve (see
+ * {@link Field}). If the set of fields contains
+ * {@link Field#LAST_EVENT_ONLY} and not {@link Field#EVENTS}, the
+ * most recent event for each entity is retrieved. If null, retrieves
+ * all fields.
+ * @return An {@link TimelineEntities} object.
+ * @throws IOException
+ */
+ TimelineEntities getEntities(String entityType,
+ Long limit, Long windowStart, Long windowEnd, String fromId, Long fromTs,
+ NameValuePair primaryFilter, Collection<NameValuePair> secondaryFilters,
+ EnumSet<Field> fieldsToRetrieve) throws IOException;
+
+ /**
+ * This method retrieves the entity information for a given entity.
+ *
+ * @param entityId
+ * The entity whose information will be retrieved.
+ * @param entityType
+ * The type of the entity.
+ * @param fieldsToRetrieve
+ * Specifies which fields of the entity object to retrieve (see
+ * {@link Field}). If the set of fields contains
+ * {@link Field#LAST_EVENT_ONLY} and not {@link Field#EVENTS}, the
+ * most recent event for each entity is retrieved. If null, retrieves
+ * all fields.
+ * @return An {@link TimelineEntity} object.
+ * @throws IOException
+ */
+ TimelineEntity getEntity(String entityId, String entityType, EnumSet<Field>
+ fieldsToRetrieve) throws IOException;
+
+ /**
+ * This method retrieves the events for a list of entities all of the same
+ * entity type. The events for each entity are sorted in order of their
+ * timestamps, descending.
+ *
+ * @param entityType
+ * The type of entities to retrieve events for.
+ * @param entityIds
+ * The entity IDs to retrieve events for.
+ * @param limit
+ * A limit on the number of events to return for each entity. If
+ * null, defaults to {@link #DEFAULT_LIMIT} events per entity.
+ * @param windowStart
+ * If not null, retrieves only events later than the given time
+ * (exclusive)
+ * @param windowEnd
+ * If not null, retrieves only events earlier than the given time
+ * (inclusive)
+ * @param eventTypes
+ * Restricts the events returned to the given types. If null, events
+ * of all types will be returned.
+ * @return An {@link TimelineEvents} object.
+ * @throws IOException
+ */
+ TimelineEvents getEntityTimelines(String entityType,
+ SortedSet<String> entityIds, Long limit, Long windowStart,
+ Long windowEnd, Set<String> eventTypes) throws IOException;
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/timeline/TimelineStore.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/timeline/TimelineStore.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/timeline/TimelineStore.java
new file mode 100644
index 0000000..3926ca1
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/timeline/TimelineStore.java
@@ -0,0 +1,29 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.metrics.timeline;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.service.Service;
+
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+public interface TimelineStore extends
+ Service, TimelineReader, TimelineWriter {
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/timeline/TimelineWriter.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/timeline/TimelineWriter.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/timeline/TimelineWriter.java
new file mode 100644
index 0000000..4b78d31
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/timeline/TimelineWriter.java
@@ -0,0 +1,46 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.metrics.timeline;
+
+import java.io.IOException;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.yarn.api.records.timeline.TimelineEntities;
+import org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse;
+
+/**
+ * This interface is for storing timeline information.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+public interface TimelineWriter {
+
+ /**
+ * Stores entity information to the timeline store. Any errors occurring for
+ * individual put request objects will be reported in the response.
+ *
+ * @param data
+ * An {@link TimelineEntities} object.
+ * @return An {@link TimelinePutResponse} object.
+ * @throws IOException
+ */
+ TimelinePutResponse put(TimelineEntities data) throws IOException;
+
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/webapp/AMSController.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/webapp/AMSController.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/webapp/AMSController.java
new file mode 100644
index 0000000..f87438c
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/webapp/AMSController.java
@@ -0,0 +1,37 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.metrics.webapp;
+
+import org.apache.hadoop.yarn.webapp.Controller;
+
+import com.google.inject.Inject;
+
+public class AMSController extends Controller {
+
+ @Inject
+ AMSController(RequestContext ctx) {
+ super(ctx);
+ }
+
+ @Override
+ public void index() {
+ setTitle("Ambari Metrics Service");
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/webapp/AMSWebApp.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/webapp/AMSWebApp.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/webapp/AMSWebApp.java
new file mode 100644
index 0000000..fa3b47c
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/webapp/AMSWebApp.java
@@ -0,0 +1,42 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.metrics.webapp;
+
+import org.apache.ambari.metrics.core.timeline.TimelineMetricStore;
+import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
+import org.apache.hadoop.yarn.webapp.WebApp;
+import org.apache.hadoop.yarn.webapp.YarnJacksonJaxbJsonProvider;
+import org.apache.hadoop.yarn.webapp.YarnWebParams;
+
+public class AMSWebApp extends WebApp implements YarnWebParams {
+
+ private final TimelineMetricStore timelineMetricStore;
+
+ public AMSWebApp(TimelineMetricStore timelineMetricStore) {
+ this.timelineMetricStore = timelineMetricStore;
+ }
+
+ @Override
+ public void setup() {
+ bind(YarnJacksonJaxbJsonProvider.class);
+ bind(TimelineWebServices.class);
+ bind(GenericExceptionHandler.class);
+ bind(TimelineMetricStore.class).toInstance(timelineMetricStore);
+ route("/", AMSController.class);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/webapp/JAXBContextResolver.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/webapp/JAXBContextResolver.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/webapp/JAXBContextResolver.java
new file mode 100644
index 0000000..404d3ea
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/webapp/JAXBContextResolver.java
@@ -0,0 +1,64 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.metrics.webapp;
+
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.Set;
+
+import javax.ws.rs.ext.ContextResolver;
+import javax.ws.rs.ext.Provider;
+import javax.xml.bind.JAXBContext;
+
+import org.apache.hadoop.yarn.server.webapp.dao.AppAttemptInfo;
+import org.apache.hadoop.yarn.server.webapp.dao.AppAttemptsInfo;
+import org.apache.hadoop.yarn.server.webapp.dao.AppInfo;
+import org.apache.hadoop.yarn.server.webapp.dao.AppsInfo;
+import org.apache.hadoop.yarn.server.webapp.dao.ContainerInfo;
+import org.apache.hadoop.yarn.server.webapp.dao.ContainersInfo;
+
+import com.google.inject.Singleton;
+import com.sun.jersey.api.json.JSONConfiguration;
+import com.sun.jersey.api.json.JSONJAXBContext;
+
+@Singleton
+@Provider
+@SuppressWarnings("rawtypes")
+public class JAXBContextResolver implements ContextResolver<JAXBContext> {
+
+ private JAXBContext context;
+ private final Set<Class> types;
+
+ // you have to specify all the dao classes here
+ private final Class[] cTypes = { AppInfo.class, AppsInfo.class,
+ AppAttemptInfo.class, AppAttemptsInfo.class, ContainerInfo.class,
+ ContainersInfo.class };
+
+ public JAXBContextResolver() throws Exception {
+ this.types = new HashSet<Class>(Arrays.asList(cTypes));
+ this.context =
+ new JSONJAXBContext(JSONConfiguration.natural().rootUnwrapping(false)
+ .build(), cTypes);
+ }
+
+ @Override
+ public JAXBContext getContext(Class<?> objectType) {
+ return (types.contains(objectType)) ? context : null;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/webapp/TimelineWebServices.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/webapp/TimelineWebServices.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/webapp/TimelineWebServices.java
new file mode 100644
index 0000000..a36b038
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/webapp/TimelineWebServices.java
@@ -0,0 +1,513 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.metrics.webapp;
+
+import java.io.IOException;
+import java.sql.SQLException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.EnumSet;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.SortedSet;
+import java.util.TreeSet;
+
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+import javax.ws.rs.Consumes;
+import javax.ws.rs.GET;
+import javax.ws.rs.POST;
+import javax.ws.rs.Path;
+import javax.ws.rs.Produces;
+import javax.ws.rs.QueryParam;
+import javax.ws.rs.WebApplicationException;
+import javax.ws.rs.core.Context;
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.Response;
+import javax.xml.bind.annotation.XmlAccessType;
+import javax.xml.bind.annotation.XmlAccessorType;
+import javax.xml.bind.annotation.XmlElement;
+import javax.xml.bind.annotation.XmlRootElement;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.metrics2.sink.timeline.AggregationResult;
+import org.apache.hadoop.metrics2.sink.timeline.ContainerMetric;
+import org.apache.hadoop.metrics2.sink.timeline.Precision;
+import org.apache.hadoop.metrics2.sink.timeline.PrecisionLimitExceededException;
+import org.apache.hadoop.metrics2.sink.timeline.TimelineMetric;
+import org.apache.hadoop.metrics2.sink.timeline.TimelineMetricMetadata;
+import org.apache.hadoop.metrics2.sink.timeline.TimelineMetrics;
+import org.apache.hadoop.metrics2.sink.timeline.TopNConfig;
+import org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse;
+import org.apache.ambari.metrics.core.timeline.TimelineMetricStore;
+import org.apache.ambari.metrics.timeline.GenericObjectMapper;
+import org.apache.ambari.metrics.timeline.NameValuePair;
+import org.apache.ambari.metrics.timeline.TimelineReader.Field;
+import org.apache.hadoop.yarn.util.timeline.TimelineUtils;
+import org.apache.hadoop.yarn.webapp.BadRequestException;
+
+import com.google.inject.Inject;
+import com.google.inject.Singleton;
+
+@Singleton
+@Path("/ws/v1/timeline")
+public class TimelineWebServices {
+ private static final Log LOG = LogFactory.getLog(TimelineWebServices.class);
+
+ private TimelineMetricStore timelineMetricStore;
+
+ @Inject
+ public TimelineWebServices(TimelineMetricStore timelineMetricStore) {
+ this.timelineMetricStore = timelineMetricStore;
+ }
+
+ @XmlRootElement(name = "about")
+ @XmlAccessorType(XmlAccessType.NONE)
+ @Public
+ @Unstable
+ public static class AboutInfo {
+
+ private String about;
+
+ public AboutInfo() {
+
+ }
+
+ public AboutInfo(String about) {
+ this.about = about;
+ }
+
+ @XmlElement(name = "About")
+ public String getAbout() {
+ return about;
+ }
+
+ public void setAbout(String about) {
+ this.about = about;
+ }
+
+ }
+
+ /**
+ * Return the description of the timeline web services.
+ */
+ @GET
+ @Produces({ MediaType.APPLICATION_JSON /* , MediaType.APPLICATION_XML */})
+ public AboutInfo about(
+ @Context HttpServletRequest req,
+ @Context HttpServletResponse res) {
+ init(res);
+ return new AboutInfo("AMS API");
+ }
+
+ /**
+ * Store the given metrics into the timeline store, and return errors that
+ * happened during storing.
+ */
+ @Path("/metrics")
+ @POST
+ @Consumes({ MediaType.APPLICATION_JSON /* , MediaType.APPLICATION_XML */})
+ public TimelinePutResponse postMetrics(
+ @Context HttpServletRequest req,
+ @Context HttpServletResponse res,
+ TimelineMetrics metrics) {
+
+ init(res);
+ if (metrics == null) {
+ return new TimelinePutResponse();
+ }
+
+ try {
+
+ // TODO: Check ACLs for MetricEntity using the TimelineACLManager.
+ // TODO: Save owner of the MetricEntity.
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Storing metrics: " +
+ TimelineUtils.dumpTimelineRecordtoJSON(metrics, true));
+ }
+
+ return timelineMetricStore.putMetrics(metrics);
+
+ } catch (Exception e) {
+ LOG.error("Error saving metrics.", e);
+ throw new WebApplicationException(e, Response.Status.INTERNAL_SERVER_ERROR);
+ }
+ }
+
+ /**
+ * Store the given metrics into the timeline store, and return errors that
+ * happened during storing.
+ */
+ @Path("/metrics/aggregated")
+ @POST
+ @Consumes({ MediaType.APPLICATION_JSON /* , MediaType.APPLICATION_XML */})
+ public TimelinePutResponse postAggregatedMetrics(
+ @Context HttpServletRequest req,
+ @Context HttpServletResponse res,
+ AggregationResult metrics) {
+
+ init(res);
+ if (metrics == null) {
+ return new TimelinePutResponse();
+ }
+
+ try {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Storing aggregated metrics: " +
+ TimelineUtils.dumpTimelineRecordtoJSON(metrics, true));
+ }
+
+ return timelineMetricStore.putHostAggregatedMetrics(metrics);
+ } catch (Exception e) {
+ LOG.error("Error saving metrics.", e);
+ throw new WebApplicationException(e, Response.Status.INTERNAL_SERVER_ERROR);
+ }
+ }
+
+ @Path("/containermetrics")
+ @POST
+ @Consumes({ MediaType.APPLICATION_JSON /* , MediaType.APPLICATION_XML */})
+ public TimelinePutResponse postContainerMetrics(
+ @Context HttpServletRequest req,
+ @Context HttpServletResponse res,
+ List<ContainerMetric> metrics) {
+ init(res);
+ if (metrics == null || metrics.isEmpty()) {
+ return new TimelinePutResponse();
+ }
+
+ try {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Storing container metrics: " + TimelineUtils
+ .dumpTimelineRecordtoJSON(metrics, true));
+ }
+
+ return timelineMetricStore.putContainerMetrics(metrics);
+
+ } catch (Exception e) {
+ LOG.error("Error saving metrics.", e);
+ throw new WebApplicationException(e, Response.Status.INTERNAL_SERVER_ERROR);
+ }
+ }
+
+ /**
+ * Query for a set of different metrics satisfying the filter criteria.
+ * All query params are optional. The default limit will apply if none
+ * specified.
+ *
+ * @param metricNames Comma separated list of metrics to retrieve.
+ * @param appId Application Id for the requested metrics.
+ * @param instanceId Application instance id.
+ * @param hostname Hostname where the metrics originated.
+ * @param startTime Start time for the metric records retrieved.
+ * @param precision Precision [ seconds, minutes, hours ]
+ * @param limit limit on total number of {@link TimelineMetric} records
+ * retrieved.
+ * @return {@link @TimelineMetrics}
+ */
+ @GET
+ @Path("/metrics")
+ @Produces({ MediaType.APPLICATION_JSON })
+ public TimelineMetrics getTimelineMetrics(
+ @Context HttpServletRequest req,
+ @Context HttpServletResponse res,
+ @QueryParam("metricNames") String metricNames,
+ @QueryParam("appId") String appId,
+ @QueryParam("instanceId") String instanceId,
+ @QueryParam("hostname") String hostname,
+ @QueryParam("startTime") String startTime,
+ @QueryParam("endTime") String endTime,
+ @QueryParam("precision") String precision,
+ @QueryParam("limit") String limit,
+ @QueryParam("grouped") String grouped,
+ @QueryParam("topN") String topN,
+ @QueryParam("topNFunction") String topNFunction,
+ @QueryParam("isBottomN") String isBottomN,
+ @QueryParam("seriesAggregateFunction") String seriesAggregateFunction
+ ) {
+ init(res);
+ try {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Request for metrics => metricNames: " + metricNames + ", " +
+ "appId: " + appId + ", instanceId: " + instanceId + ", " +
+ "hostname: " + hostname + ", startTime: " + startTime + ", " +
+ "endTime: " + endTime + ", " +
+ "precision: " + precision + "seriesAggregateFunction: " + seriesAggregateFunction);
+ }
+
+ return timelineMetricStore.getTimelineMetrics(
+ parseListStr(metricNames, ","), parseListStr(hostname, ","), appId, parseStr(instanceId),
+ parseLongStr(startTime), parseLongStr(endTime),
+ Precision.getPrecision(precision), parseIntStr(limit),
+ parseBoolean(grouped), parseTopNConfig(topN, topNFunction, isBottomN),
+ seriesAggregateFunction);
+
+ } catch (NumberFormatException ne) {
+ throw new BadRequestException("startTime and limit should be numeric " +
+ "values");
+ } catch (Precision.PrecisionFormatException pfe) {
+ throw new BadRequestException("precision should be seconds, minutes " +
+ "or hours");
+ } catch (PrecisionLimitExceededException iae) {
+ throw new PrecisionLimitExceededException(iae.getMessage());
+ } catch (IllegalArgumentException iae) {
+ throw new BadRequestException(iae.getMessage());
+ } catch (SQLException | IOException e) {
+ throw new WebApplicationException(e,
+ Response.Status.INTERNAL_SERVER_ERROR);
+ }
+ }
+
+ @GET
+ @Path("/metrics/metadata")
+ @Produces({ MediaType.APPLICATION_JSON })
+ public Map<String, List<TimelineMetricMetadata>> getTimelineMetricMetadata(
+ @Context HttpServletRequest req,
+ @Context HttpServletResponse res,
+ @QueryParam("appId") String appId,
+ @QueryParam("metricName") String metricPattern,
+ @QueryParam("includeAll") String includeBlacklistedMetrics
+ ) {
+ init(res);
+
+ try {
+ return timelineMetricStore.getTimelineMetricMetadata(
+ parseStr(appId),
+ parseStr(metricPattern),
+ parseBoolean(includeBlacklistedMetrics));
+ } catch (Exception e) {
+ throw new WebApplicationException(e, Response.Status.INTERNAL_SERVER_ERROR);
+ }
+ }
+
+ @GET
+ @Path("/metrics/hosts")
+ @Produces({ MediaType.APPLICATION_JSON })
+ public Map<String, Set<String>> getHostedAppsMetadata(
+ @Context HttpServletRequest req,
+ @Context HttpServletResponse res
+ ) {
+ init(res);
+
+ try {
+ return timelineMetricStore.getHostAppsMetadata();
+ } catch (Exception e) {
+ throw new WebApplicationException(e, Response.Status.INTERNAL_SERVER_ERROR);
+ }
+ }
+
+ @GET
+ @Path("/metrics/instances")
+ @Produces({ MediaType.APPLICATION_JSON })
+ public Map<String, Map<String, Set<String>>> getClusterHostsMetadata(
+ @Context HttpServletRequest req,
+ @Context HttpServletResponse res,
+ @QueryParam("appId") String appId,
+ @QueryParam("instanceId") String instanceId
+ ) {
+ init(res);
+
+ try {
+ return timelineMetricStore.getInstanceHostsMetadata(instanceId, appId);
+ } catch (Exception e) {
+ throw new WebApplicationException(e, Response.Status.INTERNAL_SERVER_ERROR);
+ }
+ }
+
+ @GET
+ @Path("/metrics/uuid")
+ @Produces({ MediaType.APPLICATION_JSON })
+ public byte[] getUuid(
+ @Context HttpServletRequest req,
+ @Context HttpServletResponse res,
+ @QueryParam("metricName") String metricName,
+ @QueryParam("appId") String appId,
+ @QueryParam("instanceId") String instanceId,
+ @QueryParam("hostname") String hostname
+ ) {
+ init(res);
+
+ try {
+ return timelineMetricStore.getUuid(metricName, appId, instanceId, hostname);
+ } catch (Exception e) {
+ throw new WebApplicationException(e, Response.Status.INTERNAL_SERVER_ERROR);
+ }
+ }
+
+ /**
+ * This is a discovery endpoint that advertises known live collector
+ * instances. Note: It will always answer with current instance as live.
+ * This can be utilized as a liveliness pinger endpoint since the instance
+ * names are cached and thereby no synchronous calls result from this API
+ *
+ * @return List<String> hostnames</String>
+ */
+ @GET
+ @Path("/metrics/livenodes")
+ @Produces({ MediaType.APPLICATION_JSON })
+ public List<String> getLiveCollectorNodes(
+ @Context HttpServletRequest req,
+ @Context HttpServletResponse res
+ ) {
+ init(res);
+
+ return timelineMetricStore.getLiveInstances();
+ }
+
+ private void init(HttpServletResponse response) {
+ response.setContentType(null);
+ }
+
+ private static SortedSet<String> parseArrayStr(String str, String delimiter) {
+ if (str == null) {
+ return null;
+ }
+ SortedSet<String> strSet = new TreeSet<String>();
+ String[] strs = str.split(delimiter);
+ for (String aStr : strs) {
+ strSet.add(aStr.trim());
+ }
+ return strSet;
+ }
+
+ private static NameValuePair parsePairStr(String str, String delimiter) {
+ if (str == null) {
+ return null;
+ }
+ String[] strs = str.split(delimiter, 2);
+ try {
+ return new NameValuePair(strs[0].trim(),
+ GenericObjectMapper.OBJECT_READER.readValue(strs[1].trim()));
+ } catch (Exception e) {
+ // didn't work as an Object, keep it as a String
+ return new NameValuePair(strs[0].trim(), strs[1].trim());
+ }
+ }
+
+ private static Collection<NameValuePair> parsePairsStr(
+ String str, String aDelimiter, String pDelimiter) {
+ if (str == null) {
+ return null;
+ }
+ String[] strs = str.split(aDelimiter);
+ Set<NameValuePair> pairs = new HashSet<NameValuePair>();
+ for (String aStr : strs) {
+ pairs.add(parsePairStr(aStr, pDelimiter));
+ }
+ return pairs;
+ }
+
+ private static EnumSet<Field> parseFieldsStr(String str, String delimiter) {
+ if (str == null) {
+ return null;
+ }
+ String[] strs = str.split(delimiter);
+ List<Field> fieldList = new ArrayList<Field>();
+ for (String s : strs) {
+ s = s.trim().toUpperCase();
+ if (s.equals("EVENTS")) {
+ fieldList.add(Field.EVENTS);
+ } else if (s.equals("LASTEVENTONLY")) {
+ fieldList.add(Field.LAST_EVENT_ONLY);
+ } else if (s.equals("RELATEDENTITIES")) {
+ fieldList.add(Field.RELATED_ENTITIES);
+ } else if (s.equals("PRIMARYFILTERS")) {
+ fieldList.add(Field.PRIMARY_FILTERS);
+ } else if (s.equals("OTHERINFO")) {
+ fieldList.add(Field.OTHER_INFO);
+ } else {
+ throw new IllegalArgumentException("Requested nonexistent field " + s);
+ }
+ }
+ if (fieldList.size() == 0) {
+ return null;
+ }
+ Field f1 = fieldList.remove(fieldList.size() - 1);
+ if (fieldList.size() == 0) {
+ return EnumSet.of(f1);
+ } else {
+ return EnumSet.of(f1, fieldList.toArray(new Field[fieldList.size()]));
+ }
+ }
+
+ private static Long parseLongStr(String str) {
+ return str == null ? null : Long.parseLong(str.trim());
+ }
+
+ private static Integer parseIntStr(String str) {
+ return str == null ? null : Integer.parseInt(str.trim());
+ }
+
+ private static boolean parseBoolean(String booleanStr) {
+ return booleanStr == null || Boolean.parseBoolean(booleanStr);
+ }
+
+ private static TopNConfig parseTopNConfig(String topN, String topNFunction,
+ String bottomN) {
+ if (topN == null || topN.isEmpty()) {
+ return null;
+ }
+ Integer topNValue = parseIntStr(topN);
+
+ if (topNValue == 0) {
+ LOG.info("Invalid Input for TopN query. Ignoring TopN Request.");
+ return null;
+ }
+
+ Boolean isBottomN = (bottomN != null && Boolean.parseBoolean(bottomN));
+ return new TopNConfig(topNValue, topNFunction, isBottomN);
+ }
+
+ /**
+ * Parses delimited string to list of strings. It skips strings that are
+ * effectively empty (i.e. only whitespace).
+ *
+ */
+ private static List<String> parseListStr(String str, String delimiter) {
+ if (str == null || str.trim().isEmpty()){
+ return null;
+ }
+
+ String[] split = str.trim().split(delimiter);
+ List<String> list = new ArrayList<String>(split.length);
+ for (String s : split) {
+ if (!s.trim().isEmpty()){
+ list.add(s);
+ }
+ }
+
+ return list;
+ }
+
+ private static String parseStr(String str) {
+ String trimmedInstance = (str == null) ? null : str.trim();
+ if (trimmedInstance != null) {
+ if (trimmedInstance.isEmpty() || trimmedInstance.equalsIgnoreCase("undefined")) {
+ trimmedInstance = null;
+ }
+ }
+ return trimmedInstance;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/AMSApplicationServer.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/AMSApplicationServer.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/AMSApplicationServer.java
deleted file mode 100644
index 38d46ef..0000000
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/AMSApplicationServer.java
+++ /dev/null
@@ -1,141 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.server.applicationhistoryservice;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.http.HttpConfig;
-import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
-import org.apache.hadoop.metrics2.source.JvmMetrics;
-import org.apache.hadoop.service.CompositeService;
-import org.apache.hadoop.util.ExitUtil;
-import org.apache.hadoop.util.ShutdownHookManager;
-import org.apache.hadoop.util.StringUtils;
-import org.apache.hadoop.yarn.YarnUncaughtExceptionHandler;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.HBaseTimelineMetricsService;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricStore;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.webapp.AMSWebApp;
-import org.apache.hadoop.yarn.webapp.WebApp;
-import org.apache.hadoop.yarn.webapp.WebApps;
-
-/**
- * Metrics collector web server
- */
-public class AMSApplicationServer extends CompositeService {
-
- public static final int SHUTDOWN_HOOK_PRIORITY = 30;
- private static final Log LOG = LogFactory.getLog(AMSApplicationServer.class);
-
- TimelineMetricStore timelineMetricStore;
- private WebApp webApp;
- private TimelineMetricConfiguration metricConfiguration;
-
- public AMSApplicationServer() {
- super(AMSApplicationServer.class.getName());
- }
-
- @Override
- protected void serviceInit(Configuration conf) throws Exception {
- metricConfiguration = TimelineMetricConfiguration.getInstance();
- metricConfiguration.initialize();
- timelineMetricStore = createTimelineMetricStore(conf);
- addIfService(timelineMetricStore);
- super.serviceInit(conf);
- }
-
- @Override
- protected void serviceStart() throws Exception {
- DefaultMetricsSystem.initialize("AmbariMetricsSystem");
- JvmMetrics.initSingleton("AmbariMetricsSystem", null);
-
- startWebApp();
- super.serviceStart();
- }
-
- @Override
- protected void serviceStop() throws Exception {
- if (webApp != null) {
- webApp.stop();
- }
-
- DefaultMetricsSystem.shutdown();
- super.serviceStop();
- }
-
- static AMSApplicationServer launchAMSApplicationServer(String[] args) {
- Thread.setDefaultUncaughtExceptionHandler(new YarnUncaughtExceptionHandler());
- StringUtils.startupShutdownMessage(AMSApplicationServer.class, args, LOG);
- AMSApplicationServer amsApplicationServer = null;
- try {
- amsApplicationServer = new AMSApplicationServer();
- ShutdownHookManager.get().addShutdownHook(
- new CompositeServiceShutdownHook(amsApplicationServer),
- SHUTDOWN_HOOK_PRIORITY);
- YarnConfiguration conf = new YarnConfiguration();
- amsApplicationServer.init(conf);
- amsApplicationServer.start();
- } catch (Throwable t) {
- LOG.fatal("Error starting AMSApplicationServer", t);
- ExitUtil.terminate(-1, "Error starting AMSApplicationServer");
- }
- return amsApplicationServer;
- }
-
- public static void main(String[] args) {
- launchAMSApplicationServer(args);
- }
-
- protected TimelineMetricStore createTimelineMetricStore(Configuration conf) {
- LOG.info("Creating metrics store.");
- return new HBaseTimelineMetricsService(metricConfiguration);
- }
-
- protected void startWebApp() {
- String bindAddress = null;
- try {
- bindAddress = metricConfiguration.getWebappAddress();
- } catch (Exception e) {
- throw new ExceptionInInitializerError("Cannot find bind address");
- }
- LOG.info("Instantiating metrics collector at " + bindAddress);
- try {
- Configuration conf = metricConfiguration.getMetricsConf();
- conf.set("hadoop.http.max.threads", String.valueOf(metricConfiguration
- .getTimelineMetricsServiceHandlerThreadCount()));
- HttpConfig.Policy policy = HttpConfig.Policy.valueOf(
- conf.get(TimelineMetricConfiguration.TIMELINE_SERVICE_HTTP_POLICY,
- HttpConfig.Policy.HTTP_ONLY.name()));
- webApp =
- WebApps
- .$for("timeline", null, null, "ws")
- .withHttpPolicy(conf, policy)
- .at(bindAddress)
- .start(new AMSWebApp(timelineMetricStore));
- } catch (Exception e) {
- String msg = "AHSWebApp failed to start.";
- LOG.error(msg, e);
- throw new YarnRuntimeException(msg, e);
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/LoadRunner.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/LoadRunner.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/LoadRunner.java
deleted file mode 100644
index a58ebd2..0000000
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/LoadRunner.java
+++ /dev/null
@@ -1,155 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics
- .loadsimulator;
-
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.loadsimulator.data.AppID.MASTER_APPS;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.loadsimulator.data.AppID.SLAVE_APPS;
-
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Date;
-import java.util.List;
-import java.util.concurrent.Callable;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.Future;
-import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.TimeUnit;
-
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.loadsimulator.data.AppID;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.loadsimulator.data.ApplicationInstance;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.loadsimulator.data.HostMetricsGenerator;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.loadsimulator.data.MetricsGeneratorConfigurer;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.loadsimulator.net.MetricsSender;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.loadsimulator.net.RestMetricsSender;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.loadsimulator.util.TimeStampProvider;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- *
- */
-public class LoadRunner {
- private final static Logger LOG = LoggerFactory.getLogger(LoadRunner.class);
-
- private final ScheduledExecutorService timer;
- private final ExecutorService workersPool;
- private final Collection<Callable<String>> workers;
- private final long startTime = new Date().getTime();
- private final int collectIntervalMillis;
- private final int sendIntervalMillis;
-
- public LoadRunner(String hostName,
- int threadCount,
- String metricsHostName,
- int minHostIndex,
- int collectIntervalMillis,
- int sendIntervalMillis,
- boolean createMaster) {
- this.collectIntervalMillis = collectIntervalMillis;
- this.workersPool = Executors.newFixedThreadPool(threadCount);
- this.timer = Executors.newScheduledThreadPool(1);
- this.sendIntervalMillis = sendIntervalMillis;
-
- workers = prepareWorkers(hostName, threadCount, metricsHostName, createMaster, minHostIndex);
- }
-
- private Collection<Callable<String>> prepareWorkers(String hostName,
- int threadCount,
- String metricsHost,
- Boolean createMaster, int minHostIndex) {
- Collection<Callable<String>> senderWorkers =
- new ArrayList<Callable<String>>(threadCount);
-
- int startIndex = minHostIndex;
- if (createMaster) {
- String simHost = hostName + startIndex;
- addMetricsWorkers(senderWorkers, simHost, metricsHost, MASTER_APPS);
- startIndex++;
- }
-
- for (int i = startIndex; i < threadCount + minHostIndex; i++) {
- String simHost = hostName + i;
- addMetricsWorkers(senderWorkers, simHost, metricsHost, SLAVE_APPS);
- }
-
- return senderWorkers;
- }
-
- private void addMetricsWorkers(Collection<Callable<String>> senderWorkers,
- String specificHostName,
- String metricsHostName,
- AppID[] apps) {
- for (AppID app : apps) {
- HostMetricsGenerator metricsGenerator =
- createApplicationMetrics(specificHostName, app);
- MetricsSender sender = new RestMetricsSender(metricsHostName);
- senderWorkers.add(new MetricsSenderWorker(sender, metricsGenerator));
- }
- }
-
- private HostMetricsGenerator createApplicationMetrics(String simHost, AppID host) {
- ApplicationInstance appInstance = new ApplicationInstance(simHost, host, "");
- TimeStampProvider timeStampProvider = new TimeStampProvider(startTime,
- collectIntervalMillis, sendIntervalMillis);
-
- return MetricsGeneratorConfigurer
- .createMetricsForHost(appInstance, timeStampProvider);
- }
-
- public void start() {
- timer.scheduleAtFixedRate(new Runnable() {
- @Override
- public void run() {
- try {
- runOnce();
- } catch (InterruptedException e) {
- Thread.currentThread().interrupt();
- }
- }
- }, 0, sendIntervalMillis, TimeUnit.MILLISECONDS);
- }
-
- public void runOnce() throws InterruptedException {
- List<Future<String>> futures = workersPool.invokeAll(workers,
- sendIntervalMillis / 2,
- TimeUnit.MILLISECONDS);
- int done = 0;
-
- // TODO: correctly count the failed tasks
- for (Future<String> future : futures) {
- done += future.isDone() ? 1 : 0;
- }
-
- LOG.info("Finished successfully " + done + " tasks ");
- }
-
- public void shutdown() {
- timer.shutdownNow();
- workersPool.shutdownNow();
- }
-
- public static void main(String[] args) {
- LoadRunner runner =
- new LoadRunner("local", 0, "metrics", 0, 10000, 20000, false);
-
- runner.start();
- }
-
-}
[02/28] ambari git commit: AMBARI-22740 : Fix integration test for
HBase in branch-3.0-ams due to UUID changes. (avijayan)
Posted by av...@apache.org.
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/ITMetricAggregator.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/ITMetricAggregator.java b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/ITMetricAggregator.java
deleted file mode 100644
index 1890819..0000000
--- a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/ITMetricAggregator.java
+++ /dev/null
@@ -1,355 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.aggregators;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.metrics2.sink.timeline.MetricHostAggregate;
-import org.apache.hadoop.metrics2.sink.timeline.TimelineMetric;
-import org.apache.hadoop.metrics2.sink.timeline.TimelineMetrics;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.AbstractMiniHBaseClusterTest;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.MetricTestHelper;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query.Condition;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query.DefaultCondition;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query.PhoenixTransactSQL;
-import org.junit.Test;
-
-import java.sql.PreparedStatement;
-import java.sql.ResultSet;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.TreeMap;
-
-import static junit.framework.Assert.assertEquals;
-import static junit.framework.Assert.assertTrue;
-import static junit.framework.Assert.fail;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.MetricTestHelper.createEmptyTimelineMetric;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query.PhoenixTransactSQL.GET_METRIC_AGGREGATE_ONLY_SQL;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query.PhoenixTransactSQL.METRICS_AGGREGATE_DAILY_TABLE_NAME;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query.PhoenixTransactSQL.METRICS_AGGREGATE_HOURLY_TABLE_NAME;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query.PhoenixTransactSQL.METRICS_AGGREGATE_MINUTE_TABLE_NAME;
-import static org.assertj.core.api.Assertions.assertThat;
-
-public class ITMetricAggregator extends AbstractMiniHBaseClusterTest {
-
- @Test
- public void testShouldInsertMetrics() throws Exception {
- // GIVEN
-
- // WHEN
- long startTime = System.currentTimeMillis();
- TimelineMetrics metricsSent = prepareTimelineMetrics(startTime, "local");
- hdb.insertMetricRecords(metricsSent);
-
- Condition queryCondition = new DefaultCondition(null,
- Collections.singletonList("local"), null, null, startTime,
- startTime + (15 * 60 * 1000), null, null, false);
- TimelineMetrics recordRead = hdb.getMetricRecords(queryCondition, null);
-
- // THEN
- assertThat(recordRead.getMetrics()).hasSize(2)
- .extracting("metricName")
- .containsOnly("mem_free", "disk_free");
-
- assertThat(metricsSent.getMetrics())
- .usingElementComparator(TIME_IGNORING_COMPARATOR)
- .containsExactlyElementsOf(recordRead.getMetrics());
- }
-
- private Configuration getConfigurationForTest(boolean useGroupByAggregators) {
- Configuration configuration = new Configuration();
- configuration.set("timeline.metrics.service.use.groupBy.aggregators", String.valueOf(useGroupByAggregators));
- return configuration;
- }
-
- @Test
- public void testShouldAggregateMinuteProperly() throws Exception {
- // GIVEN
- TimelineMetricAggregator aggregatorMinute =
- TimelineMetricAggregatorFactory.createTimelineMetricAggregatorMinute(hdb,
- getConfigurationForTest(false), null, null);
- TimelineMetricReadHelper readHelper = new TimelineMetricReadHelper(false);
-
- long startTime = System.currentTimeMillis();
- long ctime = startTime;
- long minute = 60 * 1000;
- hdb.insertMetricRecords(prepareTimelineMetrics(startTime, "local"));
- hdb.insertMetricRecords(prepareTimelineMetrics(ctime += minute, "local"));
- hdb.insertMetricRecords(prepareTimelineMetrics(ctime += minute, "local"));
- hdb.insertMetricRecords(prepareTimelineMetrics(ctime += minute, "local"));
- hdb.insertMetricRecords(prepareTimelineMetrics(ctime += minute, "local"));
-
- // WHEN
- long endTime = startTime + 1000 * 60 * 4;
- boolean success = aggregatorMinute.doWork(startTime, endTime);
-
- //THEN
- Condition condition = new DefaultCondition(null, null, null, null, startTime,
- endTime, null, null, true);
- condition.setStatement(String.format(GET_METRIC_AGGREGATE_ONLY_SQL,
- METRICS_AGGREGATE_MINUTE_TABLE_NAME));
-
- PreparedStatement pstmt = PhoenixTransactSQL.prepareGetMetricsSqlStmt(conn, condition);
- ResultSet rs = pstmt.executeQuery();
- MetricHostAggregate expectedAggregate =
- MetricTestHelper.createMetricHostAggregate(2.0, 0.0, 20, 15.0);
-
- int count = 0;
- while (rs.next()) {
- TimelineMetric currentMetric =
- readHelper.getTimelineMetricKeyFromResultSet(rs);
- MetricHostAggregate currentHostAggregate =
- readHelper.getMetricHostAggregateFromResultSet(rs);
-
- if ("disk_free".equals(currentMetric.getMetricName())) {
- assertEquals(2.0, currentHostAggregate.getMax());
- assertEquals(0.0, currentHostAggregate.getMin());
- assertEquals(20, currentHostAggregate.getNumberOfSamples());
- assertEquals(15.0, currentHostAggregate.getSum());
- assertEquals(15.0 / 20, currentHostAggregate.calculateAverage());
- count++;
- } else if ("mem_free".equals(currentMetric.getMetricName())) {
- assertEquals(2.0, currentHostAggregate.getMax());
- assertEquals(0.0, currentHostAggregate.getMin());
- assertEquals(20, currentHostAggregate.getNumberOfSamples());
- assertEquals(15.0, currentHostAggregate.getSum());
- assertEquals(15.0 / 20, currentHostAggregate.calculateAverage());
- count++;
- } else {
- fail("Unexpected entry");
- }
- }
- assertEquals("Two aggregated entries expected", 2, count);
- }
-
- @Test
- public void testShouldAggregateHourProperly() throws Exception {
- // GIVEN
- TimelineMetricAggregator aggregator =
- TimelineMetricAggregatorFactory.createTimelineMetricAggregatorHourly(hdb,
- getConfigurationForTest(false), null, null);
- TimelineMetricReadHelper readHelper = new TimelineMetricReadHelper(false);
- long startTime = System.currentTimeMillis();
-
- MetricHostAggregate expectedAggregate =
- MetricTestHelper.createMetricHostAggregate(2.0, 0.0, 20, 15.0);
- Map<TimelineMetric, MetricHostAggregate>
- aggMap = new HashMap<TimelineMetric,
- MetricHostAggregate>();
-
- int min_5 = 5 * 60 * 1000;
- long ctime = startTime - min_5;
- aggMap.put(createEmptyTimelineMetric(ctime += min_5), expectedAggregate);
- aggMap.put(createEmptyTimelineMetric(ctime += min_5), expectedAggregate);
- aggMap.put(createEmptyTimelineMetric(ctime += min_5), expectedAggregate);
- aggMap.put(createEmptyTimelineMetric(ctime += min_5), expectedAggregate);
- aggMap.put(createEmptyTimelineMetric(ctime += min_5), expectedAggregate);
- aggMap.put(createEmptyTimelineMetric(ctime += min_5), expectedAggregate);
- aggMap.put(createEmptyTimelineMetric(ctime += min_5), expectedAggregate);
- aggMap.put(createEmptyTimelineMetric(ctime += min_5), expectedAggregate);
- aggMap.put(createEmptyTimelineMetric(ctime += min_5), expectedAggregate);
- aggMap.put(createEmptyTimelineMetric(ctime += min_5), expectedAggregate);
- aggMap.put(createEmptyTimelineMetric(ctime += min_5), expectedAggregate);
- aggMap.put(createEmptyTimelineMetric(ctime += min_5), expectedAggregate);
-
- hdb.saveHostAggregateRecords(aggMap, METRICS_AGGREGATE_MINUTE_TABLE_NAME);
-
- //WHEN
- long endTime = ctime + min_5;
- boolean success = aggregator.doWork(startTime, endTime);
- assertTrue(success);
-
- //THEN
- Condition condition = new DefaultCondition(null, null, null, null, startTime,
- endTime, null, null, true);
- condition.setStatement(String.format(GET_METRIC_AGGREGATE_ONLY_SQL, METRICS_AGGREGATE_HOURLY_TABLE_NAME));
-
- PreparedStatement pstmt = PhoenixTransactSQL.prepareGetMetricsSqlStmt(conn, condition);
- ResultSet rs = pstmt.executeQuery();
-
- while (rs.next()) {
- TimelineMetric currentMetric =
- readHelper.getTimelineMetricKeyFromResultSet(rs);
- MetricHostAggregate currentHostAggregate =
- readHelper.getMetricHostAggregateFromResultSet(rs);
-
- if ("disk_used".equals(currentMetric.getMetricName())) {
- assertEquals(2.0, currentHostAggregate.getMax());
- assertEquals(0.0, currentHostAggregate.getMin());
- assertEquals(12 * 20, currentHostAggregate.getNumberOfSamples());
- assertEquals(12 * 15.0, currentHostAggregate.getSum());
- assertEquals(15.0 / 20, currentHostAggregate.calculateAverage());
- }
- }
- }
-
- @Test
- public void testMetricAggregateDaily() throws Exception {
- // GIVEN
- TimelineMetricAggregator aggregator =
- TimelineMetricAggregatorFactory.createTimelineMetricAggregatorDaily(hdb,
- getConfigurationForTest(false), null, null);
- TimelineMetricReadHelper readHelper = new TimelineMetricReadHelper(false);
- long startTime = System.currentTimeMillis();
-
- MetricHostAggregate expectedAggregate =
- MetricTestHelper.createMetricHostAggregate(2.0, 0.0, 20, 15.0);
- Map<TimelineMetric, MetricHostAggregate>
- aggMap = new HashMap<TimelineMetric, MetricHostAggregate>();
-
- int min_5 = 5 * 60 * 1000;
- long ctime = startTime - min_5;
- aggMap.put(createEmptyTimelineMetric(ctime += min_5), expectedAggregate);
- aggMap.put(createEmptyTimelineMetric(ctime += min_5), expectedAggregate);
- aggMap.put(createEmptyTimelineMetric(ctime += min_5), expectedAggregate);
- aggMap.put(createEmptyTimelineMetric(ctime += min_5), expectedAggregate);
- aggMap.put(createEmptyTimelineMetric(ctime += min_5), expectedAggregate);
- aggMap.put(createEmptyTimelineMetric(ctime += min_5), expectedAggregate);
- aggMap.put(createEmptyTimelineMetric(ctime += min_5), expectedAggregate);
- aggMap.put(createEmptyTimelineMetric(ctime += min_5), expectedAggregate);
- aggMap.put(createEmptyTimelineMetric(ctime += min_5), expectedAggregate);
- aggMap.put(createEmptyTimelineMetric(ctime += min_5), expectedAggregate);
- aggMap.put(createEmptyTimelineMetric(ctime += min_5), expectedAggregate);
- aggMap.put(createEmptyTimelineMetric(ctime += min_5), expectedAggregate);
-
- hdb.saveHostAggregateRecords(aggMap, METRICS_AGGREGATE_HOURLY_TABLE_NAME);
-
- //WHEN
- long endTime = ctime + min_5;
- boolean success = aggregator.doWork(startTime, endTime);
- assertTrue(success);
-
- //THEN
- Condition condition = new DefaultCondition(null, null, null, null, startTime,
- endTime, null, null, true);
- condition.setStatement(String.format(GET_METRIC_AGGREGATE_ONLY_SQL, METRICS_AGGREGATE_DAILY_TABLE_NAME));
-
- PreparedStatement pstmt = PhoenixTransactSQL.prepareGetMetricsSqlStmt(conn, condition);
- ResultSet rs = pstmt.executeQuery();
-
- while (rs.next()) {
- TimelineMetric currentMetric =
- readHelper.getTimelineMetricKeyFromResultSet(rs);
- MetricHostAggregate currentHostAggregate =
- readHelper.getMetricHostAggregateFromResultSet(rs);
-
- if ("disk_used".equals(currentMetric.getMetricName())) {
- assertEquals(2.0, currentHostAggregate.getMax());
- assertEquals(0.0, currentHostAggregate.getMin());
- assertEquals(12 * 20, currentHostAggregate.getNumberOfSamples());
- assertEquals(12 * 15.0, currentHostAggregate.getSum());
- assertEquals(15.0 / 20, currentHostAggregate.calculateAverage());
- }
- }
- }
-
- @Test
- public void testAggregationUsingGroupByQuery() throws Exception {
- // GIVEN
- TimelineMetricAggregator aggregatorMinute =
- TimelineMetricAggregatorFactory.createTimelineMetricAggregatorMinute(hdb,
- getConfigurationForTest(true), null, null);
- TimelineMetricReadHelper readHelper = new TimelineMetricReadHelper(false);
-
- long startTime = System.currentTimeMillis();
- long ctime = startTime;
- long minute = 60 * 1000;
- hdb.insertMetricRecords(prepareTimelineMetrics(startTime, "local"));
- hdb.insertMetricRecords(prepareTimelineMetrics(ctime += minute, "local"));
- hdb.insertMetricRecords(prepareTimelineMetrics(ctime += minute, "local"));
- hdb.insertMetricRecords(prepareTimelineMetrics(ctime += minute, "local"));
- hdb.insertMetricRecords(prepareTimelineMetrics(ctime += minute, "local"));
-
- long endTime = startTime + 1000 * 60 * 4;
- boolean success = aggregatorMinute.doWork(startTime, endTime);
- assertTrue(success);
-
- Condition condition = new DefaultCondition(null, null, null, null, startTime,
- endTime, null, null, true);
- condition.setStatement(String.format(GET_METRIC_AGGREGATE_ONLY_SQL, METRICS_AGGREGATE_MINUTE_TABLE_NAME));
-
- PreparedStatement pstmt = PhoenixTransactSQL.prepareGetMetricsSqlStmt(conn, condition);
- ResultSet rs = pstmt.executeQuery();
- MetricHostAggregate expectedAggregate =
- MetricTestHelper.createMetricHostAggregate(2.0, 0.0, 20, 15.0);
-
- int count = 0;
- while (rs.next()) {
- TimelineMetric currentMetric =
- readHelper.getTimelineMetricKeyFromResultSet(rs);
- MetricHostAggregate currentHostAggregate =
- readHelper.getMetricHostAggregateFromResultSet(rs);
-
- if ("disk_free".equals(currentMetric.getMetricName())) {
- assertEquals(2.0, currentHostAggregate.getMax());
- assertEquals(0.0, currentHostAggregate.getMin());
- assertEquals(20, currentHostAggregate.getNumberOfSamples());
- assertEquals(15.0, currentHostAggregate.getSum());
- assertEquals(15.0 / 20, currentHostAggregate.calculateAverage());
- count++;
- } else if ("mem_free".equals(currentMetric.getMetricName())) {
- assertEquals(2.0, currentHostAggregate.getMax());
- assertEquals(0.0, currentHostAggregate.getMin());
- assertEquals(20, currentHostAggregate.getNumberOfSamples());
- assertEquals(15.0, currentHostAggregate.getSum());
- assertEquals(15.0 / 20, currentHostAggregate.calculateAverage());
- count++;
- } else {
- fail("Unexpected entry");
- }
- }
- assertEquals("Two aggregated entries expected", 2, count);
- }
-
- private final static Comparator<TimelineMetric> TIME_IGNORING_COMPARATOR =
- new Comparator<TimelineMetric>() {
- @Override
- public int compare(TimelineMetric o1, TimelineMetric o2) {
- return o1.equalsExceptTime(o2) ? 0 : 1;
- }
- };
-
- private TimelineMetrics prepareTimelineMetrics(long startTime, String host) {
- TimelineMetrics metrics = new TimelineMetrics();
- metrics.setMetrics(Arrays.asList(
- createMetric(startTime, "disk_free", host),
- createMetric(startTime, "mem_free", host)));
-
- return metrics;
- }
-
- private TimelineMetric createMetric(long startTime, String metricName, String host) {
- TimelineMetric m = new TimelineMetric();
- m.setAppId("host");
- m.setHostName(host);
- m.setMetricName(metricName);
- m.setStartTime(startTime);
- TreeMap<Long, Double> vals = new TreeMap<Long, Double>();
- vals.put(startTime + 15000l, 0.0);
- vals.put(startTime + 30000l, 0.0);
- vals.put(startTime + 45000l, 1.0);
- vals.put(startTime + 60000l, 2.0);
-
- m.setMetricValues(vals);
-
- return m;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricClusterAggregatorSecondTest.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricClusterAggregatorSecondTest.java b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricClusterAggregatorSecondTest.java
deleted file mode 100644
index eb38625..0000000
--- a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricClusterAggregatorSecondTest.java
+++ /dev/null
@@ -1,404 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.aggregators;
-
-
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.aggregators.AggregatorUtils.getRoundedAggregateTimeMillis;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.aggregators.AggregatorUtils.getRoundedCheckPointTimeMillis;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.aggregators.AggregatorUtils.getTimeSlices;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.aggregators.AggregatorUtils.sliceFromTimelineMetric;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.availability.AggregationTaskRunner.AGGREGATOR_NAME.METRIC_AGGREGATE_SECOND;
-import static org.easymock.EasyMock.anyObject;
-import static org.easymock.EasyMock.createNiceMock;
-import static org.easymock.EasyMock.expect;
-import static org.easymock.EasyMock.replay;
-
-import java.sql.ResultSet;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.TreeMap;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.metrics2.sink.timeline.MetricClusterAggregate;
-import org.apache.hadoop.metrics2.sink.timeline.TimelineMetric;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.discovery.TimelineMetricMetadataKey;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.discovery.TimelineMetricMetadataManager;
-import org.codehaus.jackson.map.ObjectMapper;
-import org.junit.Test;
-
- import junit.framework.Assert;
-
-public class TimelineMetricClusterAggregatorSecondTest {
-
- @Test
- public void testClusterSecondAggregatorWithInterpolation() {
-
- long aggregatorInterval = 120000l;
- long sliceInterval = 30000l;
- long metricInterval = 10000l;
-
- TimelineMetricMetadataManager metricMetadataManagerMock = createNiceMock(TimelineMetricMetadataManager.class);
- expect(metricMetadataManagerMock.getUuid(anyObject(TimelineClusterMetric.class))).andReturn(new byte[16]).once();
- replay(metricMetadataManagerMock);
-
- long roundedEndTime = getRoundedAggregateTimeMillis(aggregatorInterval);
- long roundedStartTime = roundedEndTime - aggregatorInterval;
- List<Long[]> timeSlices = getTimeSlices(roundedStartTime ,
- roundedEndTime, sliceInterval);
-
- TreeMap<Long, Double> metricValues = new TreeMap<>();
-
- long startTime = roundedEndTime - aggregatorInterval;
-
- for (int i=1; startTime < roundedEndTime; i++) {
- startTime += metricInterval;
- if (i%6 == 1 || i%6 == 2) {
- metricValues.put(startTime, (double)i);
- }
- }
-
- TimelineMetric counterMetric = new TimelineMetric();
- counterMetric.setMetricName("TestMetric");
- counterMetric.setHostName("TestHost");
- counterMetric.setAppId("TestAppId");
- counterMetric.setMetricValues(metricValues);
- counterMetric.setType("COUNTER");
-
- Map<TimelineClusterMetric, Double> timelineClusterMetricMap = sliceFromTimelineMetric(counterMetric, timeSlices, true);
-
- TimelineClusterMetric timelineClusterMetric = new TimelineClusterMetric(counterMetric.getMetricName(), counterMetric.getAppId(),
- counterMetric.getInstanceId(), 0l);
-
- timelineClusterMetric.setTimestamp(roundedStartTime + 2*sliceInterval);
- Assert.assertTrue(timelineClusterMetricMap.containsKey(timelineClusterMetric));
- Assert.assertEquals(timelineClusterMetricMap.get(timelineClusterMetric), 6.0);
-
- timelineClusterMetric.setTimestamp(roundedStartTime + 4*sliceInterval);
- Assert.assertTrue(timelineClusterMetricMap.containsKey(timelineClusterMetric));
- Assert.assertEquals(timelineClusterMetricMap.get(timelineClusterMetric), 12.0);
-
- TimelineMetric metric = new TimelineMetric();
- metric.setMetricName("TestMetric");
- metric.setHostName("TestHost");
- metric.setAppId("TestAppId");
- metric.setMetricValues(metricValues);
-
- timelineClusterMetricMap = sliceFromTimelineMetric(metric, timeSlices, true);
-
- timelineClusterMetric = new TimelineClusterMetric(metric.getMetricName(), metric.getAppId(),
- metric.getInstanceId(), 0l);
-
- timelineClusterMetric.setTimestamp(roundedStartTime + 2*sliceInterval);
- Assert.assertTrue(timelineClusterMetricMap.containsKey(timelineClusterMetric));
- Assert.assertEquals(timelineClusterMetricMap.get(timelineClusterMetric), 4.5);
-
- timelineClusterMetric.setTimestamp(roundedStartTime + 4*sliceInterval);
- Assert.assertTrue(timelineClusterMetricMap.containsKey(timelineClusterMetric));
- Assert.assertEquals(timelineClusterMetricMap.get(timelineClusterMetric), 7.5);
- }
-
- @Test
- public void testShouldAggregateProperly() {
-
- long aggregatorInterval = 120000l;
- long sliceInterval = 30000l;
-
- Configuration configuration = new Configuration();
- TimelineMetricMetadataManager metricMetadataManagerMock = createNiceMock(TimelineMetricMetadataManager.class);
-
- expect(metricMetadataManagerMock.getMetadataCacheValue((TimelineMetricMetadataKey) anyObject()))
- .andReturn(null).anyTimes();
- replay(metricMetadataManagerMock);
-
- TimelineMetricClusterAggregatorSecond secondAggregator = new TimelineMetricClusterAggregatorSecond(
- METRIC_AGGREGATE_SECOND, metricMetadataManagerMock, null, configuration, null,
- aggregatorInterval, 2, "false", "", "", aggregatorInterval, sliceInterval, null
- );
-
- long startTime = getRoundedCheckPointTimeMillis(System.currentTimeMillis(),aggregatorInterval);
- List<Long[]> timeslices = getTimeSlices(startTime, startTime + aggregatorInterval, sliceInterval);
-
- Map<TimelineClusterMetric, MetricClusterAggregate> aggregateClusterMetrics = new HashMap<>();
- long seconds = 1000;
- TimelineMetric timelineMetric = new TimelineMetric();
- timelineMetric.setMetricName("m1");
- timelineMetric.setHostName("h1");
- timelineMetric.setAppId("a1");
- timelineMetric.setType("GUAGE");
- timelineMetric.setStartTime(startTime);
-
- /*
-
- 0 +30s +60s +90s +120s +150s +180s
- | | | | | | |
- (1) (2) (3) (4) (5) (6)
-
- */
- // Case 1 : Points present in all the required timeslices.
- // Life is good! Ignore (5) and (6).
-
- TreeMap<Long, Double> metricValues = new TreeMap<Long, Double>();
- metricValues.put(startTime + 15*seconds, 1.0);
- metricValues.put(startTime + 45*seconds, 2.0);
- metricValues.put(startTime + 75*seconds, 3.0);
- metricValues.put(startTime + 105*seconds, 4.0);
- metricValues.put(startTime + 135*seconds, 5.0);
- metricValues.put(startTime + 165*seconds, 6.0);
-
- timelineMetric.setMetricValues(metricValues);
- secondAggregator.processAggregateClusterMetrics(aggregateClusterMetrics, timelineMetric, timeslices);
-
- Assert.assertEquals(aggregateClusterMetrics.size(), 4);
- TimelineClusterMetric timelineClusterMetric = new TimelineClusterMetric(timelineMetric.getMetricName(),
- timelineMetric.getAppId(), timelineMetric.getInstanceId(), startTime + 30*seconds);
-
- Assert.assertTrue(aggregateClusterMetrics.containsKey(timelineClusterMetric));
- Assert.assertEquals(aggregateClusterMetrics.get(timelineClusterMetric).getSum(), 1.0);
-
- timelineClusterMetric.setTimestamp(startTime + 4*30*seconds);
- Assert.assertTrue(aggregateClusterMetrics.containsKey(timelineClusterMetric));
- Assert.assertEquals(aggregateClusterMetrics.get(timelineClusterMetric).getSum(),4.0);
-
- metricValues.clear();
- aggregateClusterMetrics.clear();
-
- /*
-
- 0 +30s +60s +90s +120s +150s +180s
- | | | | | | |
- (1) (3) (4) (5) (6)
-
- */
- // Case 2 : Some "middle" point missing in the required timeslices.
- // Interpolate the middle point. Ignore (5) and (6).
- metricValues.put(startTime + 15*seconds, 1.0);
- metricValues.put(startTime + 75*seconds, 3.0);
- metricValues.put(startTime + 105*seconds, 4.0);
- metricValues.put(startTime + 135*seconds, 5.0);
- metricValues.put(startTime + 165*seconds, 6.0);
-
- timelineMetric.setMetricValues(metricValues);
- secondAggregator.processAggregateClusterMetrics(aggregateClusterMetrics, timelineMetric, timeslices);
-
- Assert.assertEquals(aggregateClusterMetrics.size(), 4);
- timelineClusterMetric.setTimestamp(startTime + 60*seconds);
- Assert.assertTrue(aggregateClusterMetrics.containsKey(timelineClusterMetric));
- Assert.assertEquals(aggregateClusterMetrics.get(timelineClusterMetric).getSum(), 2.0);
-
- metricValues.clear();
- aggregateClusterMetrics.clear();
-
-
- /*
-
- 0 +30s +60s +90s +120s +150s +180s
- | | | | | | |
- (1) (2) (3) (5) (6)
-
- */
- // Case 3 : "end" point missing in the required timeslices.
- // Use all points to get missing point if COUNTER. Else use just (3). Ignore (6).
- metricValues.put(startTime + 15*seconds, 1.0);
- metricValues.put(startTime + 45*seconds, 2.0);
- metricValues.put(startTime + 75*seconds, 3.0);
- metricValues.put(startTime + 135*seconds, 5.0);
- metricValues.put(startTime + 165*seconds, 6.0);
-
- timelineMetric.setMetricValues(metricValues);
- timelineMetric.setType("GUAGE");
- secondAggregator.processAggregateClusterMetrics(aggregateClusterMetrics, timelineMetric, timeslices);
-
- Assert.assertEquals(aggregateClusterMetrics.size(), 4);
- timelineClusterMetric.setTimestamp(startTime + 120*seconds);
- Assert.assertTrue(aggregateClusterMetrics.containsKey(timelineClusterMetric));
- Assert.assertEquals(aggregateClusterMetrics.get(timelineClusterMetric).getSum(), 3.0);
-
- aggregateClusterMetrics.clear();
-
- timelineMetric.setType("COUNTER");
- secondAggregator.processAggregateClusterMetrics(aggregateClusterMetrics, timelineMetric, timeslices);
-
- Assert.assertEquals(aggregateClusterMetrics.size(), 4);
- timelineClusterMetric.setTimestamp(startTime + 120*seconds);
- Assert.assertTrue(aggregateClusterMetrics.containsKey(timelineClusterMetric));
- Assert.assertEquals(aggregateClusterMetrics.get(timelineClusterMetric).getSum(), 4.5);
-
- metricValues.clear();
- aggregateClusterMetrics.clear();
-
- /*
-
- 0 +30s +60s +90s +120s +150s +180s
- | | | | | | |
- (2) (3) (4) (5) (6)
-
- */
- // Case 4 : "start" point missing in the required timeslices.
- // Interpolate with only (2) to get missing point if GUAGE metric. Else use all points for COUNTER.
-
- metricValues.put(startTime + 45*seconds, 2.0);
- metricValues.put(startTime + 75*seconds, 3.0);
- metricValues.put(startTime + 105*seconds, 4.0);
- metricValues.put(startTime + 135*seconds, 5.0);
- metricValues.put(startTime + 165*seconds, 6.0);
-
- timelineMetric.setMetricValues(metricValues);
- timelineMetric.setType("GUAGE");
- secondAggregator.processAggregateClusterMetrics(aggregateClusterMetrics, timelineMetric, timeslices);
-
- Assert.assertEquals(aggregateClusterMetrics.size(), 4);
- timelineClusterMetric.setTimestamp(startTime + 30*seconds);
- Assert.assertTrue(aggregateClusterMetrics.containsKey(timelineClusterMetric));
- Assert.assertEquals(aggregateClusterMetrics.get(timelineClusterMetric).getSum(), 2.0);
-
- aggregateClusterMetrics.clear();
-
- timelineMetric.setType("COUNTER");
- int liveHosts = secondAggregator.processAggregateClusterMetrics(aggregateClusterMetrics, timelineMetric, timeslices);
-
- Assert.assertEquals(liveHosts, 1);
- Assert.assertEquals(aggregateClusterMetrics.size(), 4);
- timelineClusterMetric.setTimestamp(startTime + 30*seconds);
- Assert.assertTrue(aggregateClusterMetrics.containsKey(timelineClusterMetric));
- Assert.assertEquals(aggregateClusterMetrics.get(timelineClusterMetric).getSum(), 1.5);
-
- metricValues.clear();
- aggregateClusterMetrics.clear();
-
- /*
-
- 0 +30s +60s +90s +120s +150s +180s
- | | | | | | |
- (5) (6)
-
- */
- // Case 5 : Well, we have nothing in the 2 min window.
- // Use (5) to paint the 2 min window as (5).
-
- metricValues.put(startTime + 135*seconds, 5.0);
- metricValues.put(startTime + 165*seconds, 6.0);
-
- timelineMetric.setMetricValues(metricValues);
- timelineMetric.setType("GUAGE");
- secondAggregator.processAggregateClusterMetrics(aggregateClusterMetrics, timelineMetric, timeslices);
-
- Assert.assertEquals(aggregateClusterMetrics.size(), 4);
- timelineClusterMetric.setTimestamp(startTime + 30*seconds);
- Assert.assertTrue(aggregateClusterMetrics.containsKey(timelineClusterMetric));
- Assert.assertEquals(aggregateClusterMetrics.get(timelineClusterMetric).getSum(), 5.0);
-
- aggregateClusterMetrics.clear();
-
- timelineMetric.setType("COUNTER");
- liveHosts = secondAggregator.processAggregateClusterMetrics(aggregateClusterMetrics, timelineMetric, timeslices);
-
- Assert.assertEquals(liveHosts, 1);
- Assert.assertEquals(aggregateClusterMetrics.size(), 4);
- timelineClusterMetric.setTimestamp(startTime + 60*seconds);
- Assert.assertTrue(aggregateClusterMetrics.containsKey(timelineClusterMetric));
- Assert.assertEquals(aggregateClusterMetrics.get(timelineClusterMetric).getSum(), 2.5);
-
- metricValues.clear();
- aggregateClusterMetrics.clear();
-
- }
-
- @Test
- public void testLiveHostCounterMetrics() throws Exception {
- long aggregatorInterval = 120000;
- long sliceInterval = 30000;
-
- Configuration configuration = new Configuration();
- TimelineMetricMetadataManager metricMetadataManagerMock = createNiceMock(TimelineMetricMetadataManager.class);
-
- expect(metricMetadataManagerMock.getMetadataCacheValue((TimelineMetricMetadataKey) anyObject())).andReturn(null).anyTimes();
-
- /*
- m1-h1-a1
- m2-h1-a1
- m2-h1-a2
- m2-h2-a1
- m2-h2-a2
- m2-h3-a2
-
- So live_hosts : a1 = 2
- live_hosts : a2 = 3
- */
-
- TimelineMetric metric1 = new TimelineMetric("m1", "h1", "a1", null);
- TimelineMetric metric2 = new TimelineMetric("m2", "h1", "a1", null);
- TimelineMetric metric3 = new TimelineMetric("m2", "h1", "a2", null);
- TimelineMetric metric4 = new TimelineMetric("m2", "h2", "a1", null);
- TimelineMetric metric5 = new TimelineMetric("m2", "h2", "a2", null);
- TimelineMetric metric6 = new TimelineMetric("m2", "h3", "a2", null);
-
- expect(metricMetadataManagerMock.getMetricFromUuid((byte[]) anyObject())).
- andReturn(metric1).andReturn(metric2).andReturn(metric3).
- andReturn(metric4).andReturn(metric5).andReturn(metric6);
- replay(metricMetadataManagerMock);
-
- TimelineMetricClusterAggregatorSecond secondAggregator = new TimelineMetricClusterAggregatorSecond(
- METRIC_AGGREGATE_SECOND, metricMetadataManagerMock, null, configuration, null,
- aggregatorInterval, 2, "false", "", "", aggregatorInterval,
- sliceInterval, null);
-
- long now = System.currentTimeMillis();
- long startTime = now - 120000;
- long seconds = 1000;
- List<Long[]> slices = getTimeSlices(startTime, now, sliceInterval);
- ResultSet rs = createNiceMock(ResultSet.class);
-
- TreeMap<Long, Double> metricValues = new TreeMap<>();
- metricValues.put(startTime + 15 * seconds, 1.0);
- metricValues.put(startTime + 45 * seconds, 2.0);
- metricValues.put(startTime + 75 * seconds, 3.0);
- metricValues.put(startTime + 105 * seconds, 4.0);
-
- expect(rs.next()).andReturn(true).times(6);
- expect(rs.next()).andReturn(false);
-
- expect(rs.getLong("SERVER_TIME")).andReturn(now - 150000).times(6);
- expect(rs.getLong("START_TIME")).andReturn(now - 150000).times(6);
-
- ObjectMapper mapper = new ObjectMapper();
- expect(rs.getString("METRICS")).andReturn(mapper.writeValueAsString(metricValues)).times(6);
-
- replay(rs);
-
- Map<TimelineClusterMetric, MetricClusterAggregate> aggregates = secondAggregator.aggregateMetricsFromResultSet(rs, slices);
-
- Assert.assertNotNull(aggregates);
-
- MetricClusterAggregate a1 = null, a2 = null;
-
- for (Map.Entry<TimelineClusterMetric, MetricClusterAggregate> m : aggregates.entrySet()) {
- if (m.getKey().getMetricName().equals("live_hosts") && m.getKey().getAppId().equals("a1")) {
- a1 = m.getValue();
- }
- if (m.getKey().getMetricName().equals("live_hosts") && m.getKey().getAppId().equals("a2")) {
- a2 = m.getValue();
- }
- }
-
- Assert.assertNotNull(a1);
- Assert.assertNotNull(a2);
- Assert.assertEquals(2d, a1.getSum());
- Assert.assertEquals(3d, a2.getSum());
- }
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricClusterAggregatorSecondWithCacheSourceTest.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricClusterAggregatorSecondWithCacheSourceTest.java b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricClusterAggregatorSecondWithCacheSourceTest.java
deleted file mode 100644
index e8a9dc2..0000000
--- a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricClusterAggregatorSecondWithCacheSourceTest.java
+++ /dev/null
@@ -1,115 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.aggregators;
-
-import junit.framework.Assert;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.metrics2.sink.timeline.MetricClusterAggregate;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricsIgniteCache;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.discovery.TimelineMetricMetadataKey;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.discovery.TimelineMetricMetadataManager;
-import org.junit.BeforeClass;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.powermock.core.classloader.annotations.PowerMockIgnore;
-import org.powermock.core.classloader.annotations.PrepareForTest;
-import org.powermock.modules.junit4.PowerMockRunner;
-
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.TIMELINE_METRICS_COLLECTOR_IGNITE_NODES;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.aggregators.AggregatorUtils.getRoundedCheckPointTimeMillis;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.aggregators.AggregatorUtils.getTimeSlices;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.availability.AggregationTaskRunner.AGGREGATOR_NAME.METRIC_AGGREGATE_SECOND;
-import static org.easymock.EasyMock.anyObject;
-import static org.easymock.EasyMock.createNiceMock;
-import static org.easymock.EasyMock.expect;
-import static org.easymock.EasyMock.replay;
-import static org.powermock.api.easymock.PowerMock.mockStatic;
-import static org.powermock.api.easymock.PowerMock.replayAll;
-
-@RunWith(PowerMockRunner.class)
-@PrepareForTest(TimelineMetricConfiguration.class)
-
-@PowerMockIgnore("javax.management.*")
-public class TimelineMetricClusterAggregatorSecondWithCacheSourceTest {
-
- private static TimelineMetricsIgniteCache timelineMetricsIgniteCache;
- @BeforeClass
- public static void setupConf() throws Exception {
- TimelineMetricConfiguration conf = new TimelineMetricConfiguration(new
- Configuration(), new Configuration());
- mockStatic(TimelineMetricConfiguration.class);
- expect(TimelineMetricConfiguration.getInstance()).andReturn(conf).anyTimes();
- conf.getMetricsConf().set(TIMELINE_METRICS_COLLECTOR_IGNITE_NODES, "localhost");
- replayAll();
-
- timelineMetricsIgniteCache = new TimelineMetricsIgniteCache();
- }
-
- @Test
- public void testLiveHostCounterMetrics() throws Exception {
- long aggregatorInterval = 120000;
- long sliceInterval = 30000;
-
- Configuration configuration = new Configuration();
-
- TimelineMetricMetadataManager metricMetadataManagerMock = createNiceMock(TimelineMetricMetadataManager.class);
- expect(metricMetadataManagerMock.getMetadataCacheValue((TimelineMetricMetadataKey) anyObject())).andReturn(null).anyTimes();
- replay(metricMetadataManagerMock);
-
- TimelineMetricClusterAggregatorSecondWithCacheSource secondAggregator = new TimelineMetricClusterAggregatorSecondWithCacheSource(
- METRIC_AGGREGATE_SECOND, metricMetadataManagerMock, null, configuration, null,
- aggregatorInterval, 2, "false", "", "", aggregatorInterval,
- sliceInterval, null, timelineMetricsIgniteCache);
-
- long now = System.currentTimeMillis();
- long startTime = now - 120000;
- long seconds = 1000;
-
- Map<TimelineClusterMetric, MetricClusterAggregate> metricsFromCache = new HashMap<>();
- metricsFromCache.put(new TimelineClusterMetric("m1", "a1", "i1",startTime + 15 * seconds),
- new MetricClusterAggregate(1.0, 2, 1.0, 1.0, 1.0));
- metricsFromCache.put(new TimelineClusterMetric("m2", "a2", "i1",startTime + 18 * seconds),
- new MetricClusterAggregate(1.0, 5, 1.0, 1.0, 1.0));
-
- List<Long[]> timeslices = getTimeSlices(startTime, startTime + 120*seconds, 30*seconds);
- Map<TimelineClusterMetric, MetricClusterAggregate> aggregates = secondAggregator.aggregateMetricsFromMetricClusterAggregates(metricsFromCache, timeslices);
-
- Assert.assertNotNull(aggregates);
-
- MetricClusterAggregate a1 = null, a2 = null;
-
- for (Map.Entry<TimelineClusterMetric, MetricClusterAggregate> m : aggregates.entrySet()) {
- if (m.getKey().getMetricName().equals("live_hosts") && m.getKey().getAppId().equals("a1")) {
- a1 = m.getValue();
- }
- if (m.getKey().getMetricName().equals("live_hosts") && m.getKey().getAppId().equals("a2")) {
- a2 = m.getValue();
- }
- }
-
- Assert.assertNotNull(a1);
- Assert.assertNotNull(a2);
- Assert.assertEquals(2d, a1.getSum());
- Assert.assertEquals(5d, a2.getSum());
- }
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/availability/MetricCollectorHAControllerTest.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/availability/MetricCollectorHAControllerTest.java b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/availability/MetricCollectorHAControllerTest.java
deleted file mode 100644
index e14d069..0000000
--- a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/availability/MetricCollectorHAControllerTest.java
+++ /dev/null
@@ -1,106 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p/>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p/>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.availability;
-
-import junit.framework.Assert;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.AbstractMiniHBaseClusterTest;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration;
-import org.apache.helix.HelixManager;
-import org.apache.helix.HelixManagerFactory;
-import org.apache.helix.InstanceType;
-import org.apache.helix.model.ExternalView;
-import org.apache.helix.model.InstanceConfig;
-import org.junit.Before;
-import org.junit.Test;
-import java.util.HashMap;
-import java.util.Map;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.availability.MetricCollectorHAController.DEFAULT_STATE_MODEL;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.availability.MetricCollectorHAController.METRIC_AGGREGATORS;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.availability.MetricCollectorHAController.CLUSTER_NAME;
-import static org.easymock.EasyMock.createNiceMock;
-import static org.easymock.EasyMock.expect;
-import static org.easymock.EasyMock.replay;
-
-public class MetricCollectorHAControllerTest extends AbstractMiniHBaseClusterTest {
- TimelineMetricConfiguration configuration;
-
- @Before
- public void setup() throws Exception {
- configuration = createNiceMock(TimelineMetricConfiguration.class);
-
- expect(configuration.getInstanceHostnameFromEnv()).andReturn("h1");
- expect(configuration.getInstancePort()).andReturn("12000");
- // jdbc:phoenix:localhost:52887:/hbase;test=true
- String zkUrl = getUrl();
- String port = zkUrl.split(":")[3];
- String quorum = zkUrl.split(":")[2];
-
- expect(configuration.getClusterZKClientPort()).andReturn(port);
- expect(configuration.getClusterZKQuorum()).andReturn(quorum);
- expect(configuration.getZkConnectionUrl(port, quorum)).andReturn(quorum + ":" + port);
-
- replay(configuration);
- }
-
- @Test(timeout = 180000)
- public void testHAControllerDistributedAggregation() throws Exception {
- MetricCollectorHAController haController = new MetricCollectorHAController(configuration);
- haController.initializeHAController();
- // Wait for task assignment
- Thread.sleep(10000);
-
- Assert.assertTrue(haController.isInitialized());
- Assert.assertEquals(1, haController.getLiveInstanceHostNames().size());
- Assert.assertTrue(haController.getAggregationTaskRunner().performsClusterAggregation());
- Assert.assertTrue(haController.getAggregationTaskRunner().performsHostAggregation());
-
- // Add new instance
- InstanceConfig instanceConfig2 = new InstanceConfig("h2_12001");
- haController.admin.addInstance(CLUSTER_NAME, instanceConfig2);
- HelixManager manager2 = HelixManagerFactory.getZKHelixManager(CLUSTER_NAME,
- instanceConfig2.getInstanceName(),
- InstanceType.PARTICIPANT, haController.zkConnectUrl);
- manager2.getStateMachineEngine().registerStateModelFactory(DEFAULT_STATE_MODEL,
- new OnlineOfflineStateModelFactory(instanceConfig2.getInstanceName(),
- new AggregationTaskRunner(instanceConfig2.getInstanceName(), "", CLUSTER_NAME)));
- manager2.connect();
- haController.admin.rebalance(CLUSTER_NAME, METRIC_AGGREGATORS, 1);
-
- // Wait on re-assignment of partitions
- Thread.sleep(10000);
- Assert.assertEquals(2, haController.getLiveInstanceHostNames().size());
-
- ExternalView view = haController.admin.getResourceExternalView(CLUSTER_NAME, METRIC_AGGREGATORS);
-
- Map<String, String> partitionInstanceMap = new HashMap<>();
-
- for (String partition : view.getPartitionSet()) {
- Map<String, String> states = view.getStateMap(partition);
- // (instance, state) pairs
- for (Map.Entry<String, String> stateEntry : states.entrySet()) {
- partitionInstanceMap.put(partition, stateEntry.getKey());
- Assert.assertEquals("ONLINE", stateEntry.getValue());
- }
- }
- // Re-assigned partitions
- Assert.assertEquals(2, partitionInstanceMap.size());
-
- haController.getAggregationTaskRunner().stop();
- haController.manager.disconnect();
- }
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/discovery/TestMetadataManager.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/discovery/TestMetadataManager.java b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/discovery/TestMetadataManager.java
deleted file mode 100644
index 3e3b91f..0000000
--- a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/discovery/TestMetadataManager.java
+++ /dev/null
@@ -1,234 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p/>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p/>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.discovery;
-
-import static org.easymock.EasyMock.createNiceMock;
-import static org.easymock.EasyMock.expect;
-import static org.easymock.EasyMock.replay;
-
-import junit.framework.Assert;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.metrics2.sink.timeline.TimelineMetric;
-import org.apache.hadoop.metrics2.sink.timeline.TimelineMetricMetadata;
-import org.apache.hadoop.metrics2.sink.timeline.TimelineMetrics;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.AbstractMiniHBaseClusterTest;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.aggregators.TimelineClusterMetric;
-import org.easymock.EasyMock;
-import org.junit.Before;
-import org.junit.Test;
-
-import java.io.IOException;
-import java.net.URISyntaxException;
-import java.sql.SQLException;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.TreeMap;
-
-public class TestMetadataManager extends AbstractMiniHBaseClusterTest {
- TimelineMetricMetadataManager metadataManager;
-
- @Before
- public void insertDummyRecords() throws IOException, SQLException, URISyntaxException {
-
- final long now = System.currentTimeMillis();
-
- TimelineMetrics timelineMetrics = new TimelineMetrics();
- TimelineMetric metric1 = new TimelineMetric();
- metric1.setMetricName("dummy_metric1");
- metric1.setHostName("dummy_host1");
- metric1.setStartTime(now - 1000);
- metric1.setAppId("dummy_app1");
- metric1.setType("Integer");
- metric1.setMetricValues(new TreeMap<Long, Double>() {{
- put(now - 100, 1.0);
- put(now - 200, 2.0);
- put(now - 300, 3.0);
- }});
- timelineMetrics.getMetrics().add(metric1);
- TimelineMetric metric2 = new TimelineMetric();
- metric2.setMetricName("dummy_metric2");
- metric2.setHostName("dummy_host2");
- metric2.setStartTime(now - 1000);
- metric2.setAppId("dummy_app2");
- metric2.setType("Integer");
- metric2.setInstanceId("instance2");
- metric2.setMetricValues(new TreeMap<Long, Double>() {{
- put(now - 100, 1.0);
- put(now - 200, 2.0);
- put(now - 300, 3.0);
- }});
- timelineMetrics.getMetrics().add(metric2);
-
- Configuration metricsConf = createNiceMock(Configuration.class);
- expect(metricsConf.get("timeline.metrics.service.operation.mode")).andReturn("distributed").anyTimes();
- replay(metricsConf);
-
- // Initialize new manager
- metadataManager = new TimelineMetricMetadataManager(metricsConf, hdb);
- hdb.setMetadataInstance(metadataManager);
-
- hdb.insertMetricRecordsWithMetadata(metadataManager, timelineMetrics, true);
- }
-
- @Test(timeout = 180000)
- public void testSaveMetricsMetadata() throws Exception {
- Map<TimelineMetricMetadataKey, TimelineMetricMetadata> cachedData = metadataManager.getMetadataCache();
-
- Assert.assertNotNull(cachedData);
- Assert.assertEquals(2, cachedData.size());
- TimelineMetricMetadataKey key1 = new TimelineMetricMetadataKey("dummy_metric1", "dummy_app1", null);
- TimelineMetricMetadataKey key2 = new TimelineMetricMetadataKey("dummy_metric2", "dummy_app2", "instance2");
- TimelineMetricMetadata value1 = new TimelineMetricMetadata("dummy_metric1",
- "dummy_app1", null, null, "Integer", 1L, true, true);
- TimelineMetricMetadata value2 = new TimelineMetricMetadata("dummy_metric2",
- "dummy_app2", "instance2", null, "Integer", 1L, true, true);
-
- Assert.assertEquals(value1, cachedData.get(key1));
- Assert.assertEquals(value2, cachedData.get(key2));
-
- TimelineMetricMetadataSync syncRunnable = new TimelineMetricMetadataSync(metadataManager);
- syncRunnable.run();
-
- Map<TimelineMetricMetadataKey, TimelineMetricMetadata> savedData =
- hdb.getTimelineMetricMetadata();
-
- Assert.assertNotNull(savedData);
- Assert.assertEquals(2, savedData.size());
- Assert.assertEquals(value1, savedData.get(key1));
- Assert.assertEquals(value2, savedData.get(key2));
-
- Map<String, TimelineMetricHostMetadata> cachedHostData = metadataManager.getHostedAppsCache();
- Map<String, TimelineMetricHostMetadata> savedHostData = metadataManager.getHostedAppsFromStore();
- Assert.assertEquals(cachedData.size(), savedData.size());
- Assert.assertEquals("dummy_app1", cachedHostData.get("dummy_host1").getHostedApps().keySet().iterator().next());
- Assert.assertEquals("dummy_app2", cachedHostData.get("dummy_host2").getHostedApps().keySet().iterator().next());
- Assert.assertEquals("dummy_app1", savedHostData.get("dummy_host1").getHostedApps().keySet().iterator().next());
- Assert.assertEquals("dummy_app2", savedHostData.get("dummy_host2").getHostedApps().keySet().iterator().next());
-
- Map<String, Set<String>> cachedHostInstanceData = metadataManager.getHostedInstanceCache();
- Map<String, Set<String>> savedHostInstanceData = metadataManager.getHostedInstancesFromStore();
- Assert.assertEquals(cachedHostInstanceData.size(), savedHostInstanceData.size());
- Assert.assertEquals("dummy_host2", cachedHostInstanceData.get("instance2").iterator().next());
- }
-
- @Test
- public void testGenerateUuidFromMetric() throws SQLException {
-
- TimelineMetric timelineMetric = new TimelineMetric();
- timelineMetric.setMetricName("regionserver.Server.blockCacheExpressHitPercent");
- timelineMetric.setAppId("hbase");
- timelineMetric.setHostName("avijayan-ams-2.openstacklocal");
- timelineMetric.setInstanceId("test1");
-
- byte[] uuid = metadataManager.getUuid(timelineMetric);
- Assert.assertNotNull(uuid);
- Assert.assertEquals(uuid.length, 20);
-
- byte[] uuidWithoutHost = metadataManager.getUuid(new TimelineClusterMetric(timelineMetric.getMetricName(), timelineMetric.getAppId(), timelineMetric.getInstanceId(), -1));
- Assert.assertNotNull(uuidWithoutHost);
- Assert.assertEquals(uuidWithoutHost.length, 16);
-
- TimelineMetric metric2 = metadataManager.getMetricFromUuid(uuid);
- Assert.assertEquals(metric2, timelineMetric);
- TimelineMetric metric3 = metadataManager.getMetricFromUuid(uuidWithoutHost);
- Assert.assertEquals(metric3.getMetricName(), timelineMetric.getMetricName());
- Assert.assertEquals(metric3.getAppId(), timelineMetric.getAppId());
- Assert.assertEquals(metric3.getInstanceId(), timelineMetric.getInstanceId());
- Assert.assertEquals(metric3.getHostName(), null);
-
- String metricName1 = metadataManager.getMetricNameFromUuid(uuid);
- Assert.assertEquals(metricName1, "regionserver.Server.blockCacheExpressHitPercent");
- String metricName2 = metadataManager.getMetricNameFromUuid(uuidWithoutHost);
- Assert.assertEquals(metricName2, "regionserver.Server.blockCacheExpressHitPercent");
- }
-
- @Test
- public void testWildcardSanitization() throws IOException, SQLException, URISyntaxException {
- // Initialize new manager
- metadataManager = new TimelineMetricMetadataManager(new Configuration(), hdb);
- final long now = System.currentTimeMillis();
-
- TimelineMetrics timelineMetrics = new TimelineMetrics();
-
- TimelineMetric metric1 = new TimelineMetric();
- metric1.setMetricName("dummy_m1");
- metric1.setHostName("dummy_host1");
- metric1.setStartTime(now - 1000);
- metric1.setAppId("dummy_app1");
- metric1.setType("Integer");
- metric1.setMetricValues(new TreeMap<Long, Double>() {{
- put(now - 100, 1.0);
- put(now - 200, 2.0);
- put(now - 300, 3.0);
- }});
- timelineMetrics.getMetrics().add(metric1);
-
- TimelineMetric metric2 = new TimelineMetric();
- metric2.setMetricName("dummy_m2");
- metric2.setHostName("dummy_host2");
- metric2.setStartTime(now - 1000);
- metric2.setAppId("dummy_app2");
- metric2.setType("Integer");
- metric2.setMetricValues(new TreeMap<Long, Double>() {{
- put(now - 100, 1.0);
- put(now - 200, 2.0);
- put(now - 300, 3.0);
- }});
- timelineMetrics.getMetrics().add(metric2);
-
- TimelineMetric metric3 = new TimelineMetric();
- metric3.setMetricName("gummy_3");
- metric3.setHostName("dummy_3h");
- metric3.setStartTime(now - 1000);
- metric3.setAppId("dummy_app3");
- metric3.setType("Integer");
- metric3.setMetricValues(new TreeMap<Long, Double>() {{
- put(now - 100, 1.0);
- put(now - 200, 2.0);
- put(now - 300, 3.0);
- }});
- timelineMetrics.getMetrics().add(metric3);
-
- Configuration metricsConf = new Configuration();
- TimelineMetricConfiguration configuration = EasyMock.createNiceMock(TimelineMetricConfiguration.class);
- expect(configuration.getMetricsConf()).andReturn(metricsConf).once();
- replay(configuration);
-
- hdb.insertMetricRecordsWithMetadata(metadataManager, timelineMetrics, true);
-
- List<byte[]> uuids = metadataManager.getUuids(Collections.singletonList("dummy_m%"),
- Collections.singletonList("dummy_host2"), "dummy_app1", null);
- Assert.assertTrue(uuids.size() == 2);
-
- uuids = metadataManager.getUuids(Collections.singletonList("dummy_m%"),
- Collections.singletonList("dummy_host%"), "dummy_app2", null);
- Assert.assertTrue(uuids.size() == 4);
-
- Collection<String> metrics = Arrays.asList("dummy_m%", "dummy_3", "dummy_m2");
- List<String> hosts = Arrays.asList("dummy_host%", "dummy_3h");
- uuids = metadataManager.getUuids(metrics, hosts, "dummy_app2", null);
- Assert.assertTrue(uuids.size() == 9);
- }
-
-
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/discovery/TestMetadataSync.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/discovery/TestMetadataSync.java b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/discovery/TestMetadataSync.java
deleted file mode 100644
index 8d486e1..0000000
--- a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/discovery/TestMetadataSync.java
+++ /dev/null
@@ -1,126 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p/>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p/>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.discovery;
-
-import junit.framework.Assert;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.metrics2.sink.timeline.TimelineMetricMetadata;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.PhoenixHBaseAccessor;
-import org.junit.Test;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Map;
-import java.util.Set;
-import static org.apache.hadoop.metrics2.sink.timeline.TimelineMetricMetadata.MetricType.GAUGE;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.TIMELINE_METRIC_METADATA_FILTERS;
-import static org.easymock.EasyMock.createNiceMock;
-import static org.easymock.EasyMock.expect;
-import static org.easymock.EasyMock.replay;
-import static org.easymock.EasyMock.verify;
-
-public class TestMetadataSync {
- @Test
- public void testRefreshMetadataOnWrite() throws Exception {
- Configuration configuration = createNiceMock(Configuration.class);
- PhoenixHBaseAccessor hBaseAccessor = createNiceMock(PhoenixHBaseAccessor.class);
-
- final TimelineMetricMetadata testMetadata1 = new TimelineMetricMetadata(
- "m1", "a1", null, "", GAUGE.name(), System.currentTimeMillis(), true, false);
- final TimelineMetricMetadata testMetadata2 = new TimelineMetricMetadata(
- "m2", "a2", null, "", GAUGE.name(), System.currentTimeMillis(), true, false);
-
- Map<TimelineMetricMetadataKey, TimelineMetricMetadata> metadata =
- new HashMap<TimelineMetricMetadataKey, TimelineMetricMetadata>() {{
- put(new TimelineMetricMetadataKey("m1", "a1", null), testMetadata1);
- put(new TimelineMetricMetadataKey("m2", "a2", null), testMetadata2);
- }};
-
- Map<String, TimelineMetricHostMetadata> hostedApps = new HashMap<String, TimelineMetricHostMetadata>() {{
- put("h1", new TimelineMetricHostMetadata(new HashSet<>(Arrays.asList("a1"))));
- put("h2", new TimelineMetricHostMetadata((new HashSet<>(Arrays.asList("a1", "a2")))));
- }};
-
- Map<String, Set<String>> hostedInstances = new HashMap<String, Set<String>>() {{
- put("i1", new HashSet<>(Arrays.asList("h1")));
- put("i2", new HashSet<>(Arrays.asList("h1", "h2")));
- }};
-
- expect(configuration.get("timeline.metrics.service.operation.mode")).andReturn("distributed");
- expect(hBaseAccessor.getTimelineMetricMetadata()).andReturn(metadata);
- expect(hBaseAccessor.getHostedAppsMetadata()).andReturn(hostedApps);
- expect(hBaseAccessor.getInstanceHostsMetdata()).andReturn(hostedInstances);
-
- replay(configuration, hBaseAccessor);
-
- TimelineMetricMetadataManager metadataManager = new TimelineMetricMetadataManager(configuration, hBaseAccessor);
-
- metadataManager.metricMetadataSync = new TimelineMetricMetadataSync(metadataManager);
-
- metadataManager.metricMetadataSync.run();
-
- verify(configuration, hBaseAccessor);
-
- metadata = metadataManager.getMetadataCache();
- Assert.assertEquals(2, metadata.size());
- Assert.assertTrue(metadata.containsKey(new TimelineMetricMetadataKey("m1", "a1", null)));
- Assert.assertTrue(metadata.containsKey(new TimelineMetricMetadataKey("m2", "a2", null)));
-
- hostedApps = metadataManager.getHostedAppsCache();
- Assert.assertEquals(2, hostedApps.size());
- Assert.assertEquals(1, hostedApps.get("h1").getHostedApps().size());
- Assert.assertEquals(2, hostedApps.get("h2").getHostedApps().size());
-
- hostedInstances = metadataManager.getHostedInstanceCache();
- Assert.assertEquals(2, hostedInstances.size());
- Assert.assertEquals(1, hostedInstances.get("i1").size());
- Assert.assertEquals(2, hostedInstances.get("i2").size());
-
- }
-
- @Test
- public void testFilterByRegexOnMetricName() throws Exception {
- Configuration configuration = createNiceMock(Configuration.class);
- PhoenixHBaseAccessor hBaseAccessor = createNiceMock(PhoenixHBaseAccessor.class);
-
- TimelineMetricMetadata metadata1 = new TimelineMetricMetadata(
- "xxx.abc.yyy", "a1", null, "", GAUGE.name(), System.currentTimeMillis(), true, false);
- TimelineMetricMetadata metadata2 = new TimelineMetricMetadata(
- "xxx.cdef.yyy", "a2", null, "", GAUGE.name(), System.currentTimeMillis(), true, false);
- TimelineMetricMetadata metadata3 = new TimelineMetricMetadata(
- "xxx.pqr.zzz", "a3", null, "", GAUGE.name(), System.currentTimeMillis(), true, false);
-
- expect(configuration.get(TIMELINE_METRIC_METADATA_FILTERS)).andReturn("abc,cde");
-
- replay(configuration, hBaseAccessor);
-
- TimelineMetricMetadataManager metadataManager = new TimelineMetricMetadataManager(configuration, hBaseAccessor);
-
- metadataManager.putIfModifiedTimelineMetricMetadata(metadata1);
- metadataManager.putIfModifiedTimelineMetricMetadata(metadata2);
- metadataManager.putIfModifiedTimelineMetricMetadata(metadata3);
-
- verify(configuration, hBaseAccessor);
-
- Map<TimelineMetricMetadataKey, TimelineMetricMetadata> metadata =
- metadataManager.getMetadataCache();
-
- Assert.assertEquals(1, metadata.size());
- Assert.assertEquals("xxx.pqr.zzz", metadata.keySet().iterator().next().getMetricName());
- }
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/function/TimelineMetricsSeriesAggregateFunctionTest.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/function/TimelineMetricsSeriesAggregateFunctionTest.java b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/function/TimelineMetricsSeriesAggregateFunctionTest.java
deleted file mode 100644
index bd54bad..0000000
--- a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/function/TimelineMetricsSeriesAggregateFunctionTest.java
+++ /dev/null
@@ -1,188 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.function;
-
-import com.google.common.collect.Lists;
-import org.apache.hadoop.metrics2.sink.timeline.TimelineMetric;
-import org.apache.hadoop.metrics2.sink.timeline.TimelineMetrics;
-import org.junit.Test;
-
-import java.util.Map;
-import java.util.TreeMap;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
-public class TimelineMetricsSeriesAggregateFunctionTest {
-
- public static final double DELTA = 0.00000000001;
-
- @Test public void testSeriesAggregateBySummation() throws Exception {
- TimelineMetrics testMetrics = getTestObject();
- // all TimelineMetric are having same values
- TreeMap<Long, Double> metricValues = testMetrics.getMetrics().get(0).getMetricValues();
-
- TimelineMetricsSeriesAggregateFunction function = TimelineMetricsSeriesAggregateFunctionFactory
- .newInstance(SeriesAggregateFunction.SUM);
- TimelineMetric aggregatedMetric = function.apply(testMetrics);
-
- String aggregatedMetricName = aggregatedMetric.getMetricName();
- String aggregatedHostName = aggregatedMetric.getHostName();
- String aggregatedAppId = aggregatedMetric.getAppId();
- String aggregatedInstanceId = aggregatedMetric.getInstanceId();
-
- for (TimelineMetric testMetric : testMetrics.getMetrics()) {
- assertTrue(aggregatedMetricName.contains(testMetric.getMetricName()));
- assertTrue(aggregatedHostName.contains(testMetric.getHostName()));
- if (!testMetric.getMetricName().equals("byte_in.3")) {
- assertTrue(aggregatedAppId.contains(testMetric.getAppId()));
- assertTrue(aggregatedInstanceId.contains(testMetric.getInstanceId()));
- }
- }
-
- TreeMap<Long, Double> summationMetricValues = aggregatedMetric.getMetricValues();
- assertEquals(3, summationMetricValues.size());
- for (Map.Entry<Long, Double> tsAndValue : summationMetricValues.entrySet()) {
- assertEquals(metricValues.get(tsAndValue.getKey()) * 3, tsAndValue.getValue(), DELTA);
- }
- }
-
- @Test public void testSeriesAggregateByAverage() throws Exception {
- TimelineMetrics testMetrics = getTestObject();
- // all TimelineMetric are having same values
- TreeMap<Long, Double> metricValues = testMetrics.getMetrics().get(0).getMetricValues();
-
- TimelineMetricsSeriesAggregateFunction function = TimelineMetricsSeriesAggregateFunctionFactory
- .newInstance(SeriesAggregateFunction.AVG);
- TimelineMetric aggregatedMetric = function.apply(testMetrics);
-
- // checks only values, others are covered by testSeriesAggregateBySummation
- TreeMap<Long, Double> averageMetricValues = aggregatedMetric.getMetricValues();
- assertEquals(3, averageMetricValues.size());
- for (Map.Entry<Long, Double> tsAndValue : averageMetricValues.entrySet()) {
- assertEquals(metricValues.get(tsAndValue.getKey()), tsAndValue.getValue(), DELTA);
- }
- }
-
- @Test public void testSeriesAggregateByMax() throws Exception {
- TimelineMetrics testMetrics = getTestObject();
-
- // override metric values
- TreeMap<Long, Double> metricValues = new TreeMap<>();
- metricValues.put(1L, 1.0);
- metricValues.put(2L, 2.0);
- metricValues.put(3L, 3.0);
-
- testMetrics.getMetrics().get(0).setMetricValues(metricValues);
-
- TreeMap<Long, Double> metricValues2 = new TreeMap<>();
- metricValues2.put(1L, 2.0);
- metricValues2.put(2L, 4.0);
- metricValues2.put(3L, 6.0);
-
- testMetrics.getMetrics().get(1).setMetricValues(metricValues2);
-
- TreeMap<Long, Double> metricValues3 = new TreeMap<>();
- metricValues3.put(1L, 3.0);
- metricValues3.put(2L, 6.0);
- metricValues3.put(3L, 9.0);
-
- testMetrics.getMetrics().get(2).setMetricValues(metricValues3);
-
- TimelineMetricsSeriesAggregateFunction function = TimelineMetricsSeriesAggregateFunctionFactory
- .newInstance(SeriesAggregateFunction.MAX);
- TimelineMetric aggregatedMetric = function.apply(testMetrics);
-
- // checks only values, others are covered by testSeriesAggregateBySummation
- TreeMap<Long, Double> maxMetricValues = aggregatedMetric.getMetricValues();
- assertEquals(3, maxMetricValues.size());
- for (Map.Entry<Long, Double> tsAndValue : maxMetricValues.entrySet()) {
- assertEquals(metricValues3.get(tsAndValue.getKey()), tsAndValue.getValue(), DELTA);
- }
- }
-
- @Test public void testSeriesAggregateByMin() throws Exception {
- TimelineMetrics testMetrics = getTestObject();
-
- // override metric values
- TreeMap<Long, Double> metricValues = new TreeMap<>();
- metricValues.put(1L, 1.0);
- metricValues.put(2L, 2.0);
- metricValues.put(3L, 3.0);
-
- testMetrics.getMetrics().get(0).setMetricValues(metricValues);
-
- TreeMap<Long, Double> metricValues2 = new TreeMap<>();
- metricValues2.put(1L, 2.0);
- metricValues2.put(2L, 4.0);
- metricValues2.put(3L, 6.0);
-
- testMetrics.getMetrics().get(1).setMetricValues(metricValues2);
-
- TreeMap<Long, Double> metricValues3 = new TreeMap<>();
- metricValues3.put(1L, 3.0);
- metricValues3.put(2L, 6.0);
- metricValues3.put(3L, 9.0);
-
- testMetrics.getMetrics().get(2).setMetricValues(metricValues3);
-
- TimelineMetricsSeriesAggregateFunction function = TimelineMetricsSeriesAggregateFunctionFactory
- .newInstance(SeriesAggregateFunction.MIN);
- TimelineMetric aggregatedMetric = function.apply(testMetrics);
-
- // checks only values, others are covered by testSeriesAggregateBySummation
- TreeMap<Long, Double> minMetricValues = aggregatedMetric.getMetricValues();
- assertEquals(3, minMetricValues.size());
- for (Map.Entry<Long, Double> tsAndValue : minMetricValues.entrySet()) {
- assertEquals(metricValues.get(tsAndValue.getKey()), tsAndValue.getValue(), DELTA);
- }
- }
-
- private TimelineMetrics getTestObject() {
- TimelineMetric metric = new TimelineMetric();
- metric.setMetricName("byte_in.1");
- metric.setHostName("host1");
- metric.setAppId("app1");
- metric.setInstanceId("instance1");
-
- TimelineMetric metric2 = new TimelineMetric();
- metric2.setMetricName("byte_in.2");
- metric2.setHostName("host2");
- metric2.setAppId("app2");
- metric2.setInstanceId("instance2");
-
- TimelineMetric metric3 = new TimelineMetric();
- metric3.setMetricName("byte_in.3");
- metric3.setHostName("host3");
- // appId and instanceId for metric3 are null
-
- TreeMap<Long, Double> metricValues = new TreeMap<>();
- metricValues.put(1L, 3.0);
- metricValues.put(2L, 2 * 3.0);
- metricValues.put(3L, 3 * 3.0);
-
- metric.setMetricValues(metricValues);
- metric2.setMetricValues(metricValues);
- metric3.setMetricValues(metricValues);
-
- TimelineMetrics metrics = new TimelineMetrics();
- metrics.setMetrics(Lists.newArrayList(metric, metric2, metric3));
-
- return metrics;
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/query/DefaultConditionTest.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/query/DefaultConditionTest.java b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/query/DefaultConditionTest.java
deleted file mode 100644
index 71226e8..0000000
--- a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/query/DefaultConditionTest.java
+++ /dev/null
@@ -1,170 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query;
-
-import junit.framework.Assert;
-import org.apache.commons.collections.CollectionUtils;
-import org.junit.Test;
-
-import java.util.ArrayList;
-import java.util.List;
-
-public class DefaultConditionTest {
-
- @Test
- public void testMetricNameWhereCondition() {
- //EMPTY
- List<byte[]> uuids = new ArrayList<>();
- DefaultCondition condition = new DefaultCondition(uuids,null,null,null,null,null,null,null,null,true);
- StringBuilder sb = new StringBuilder();
- condition.appendUuidClause(sb);
- Assert.assertEquals(sb.toString(), "");
- Assert.assertTrue(CollectionUtils.isEqualCollection(uuids, condition.getUuids()));
-
- //Metric uuid
- uuids.add(new byte[16]);
- condition = new DefaultCondition(uuids,null,null,null,null,null,null,null,null,true);
- sb = new StringBuilder();
- condition.appendUuidClause(sb);
- Assert.assertEquals(sb.toString(), "(UUID LIKE ?)");
- Assert.assertEquals(uuids.size(), condition.getUuids().size());
- Assert.assertTrue(new String(condition.getUuids().get(0)).endsWith("%"));
-
- //metric uuid + Host uuid
- uuids.add(new byte[4]);
- condition = new DefaultCondition(uuids,null,null,null,null,null,null,null,null,true);
- sb = new StringBuilder();
- condition.appendUuidClause(sb);
- Assert.assertEquals(sb.toString(), "(UUID LIKE ? AND UUID LIKE ?)");
- Assert.assertEquals(uuids.size(), condition.getUuids().size());
- Assert.assertTrue(new String(condition.getUuids().get(1)).startsWith("%"));
-
- //metric + host + full
- uuids.add(new byte[20]);
- uuids.add(new byte[20]);
- condition = new DefaultCondition(uuids,null,null,null,null,null,null,null,null,true);
- sb = new StringBuilder();
- condition.appendUuidClause(sb);
- Assert.assertEquals(sb.toString(), "(UUID IN (?, ?) AND UUID LIKE ? AND UUID LIKE ?)");
- Assert.assertEquals(uuids.size(), condition.getUuids().size());
-
- //Only IN clause.
- uuids.clear();
- uuids.add(new byte[20]);
- condition = new DefaultCondition(uuids,null,null,null,null,null,null,null,null,true);
- sb = new StringBuilder();
- condition.appendUuidClause(sb);
- Assert.assertEquals(sb.toString(), "(UUID IN (?))");
- Assert.assertEquals(uuids.size(), condition.getUuids().size());
-
- //metric NOT LIKE
- uuids.clear();
- uuids.add(new byte[16]);
- condition = new DefaultCondition(uuids,null,null,null,null,null,null,null,null,true);
- sb = new StringBuilder();
- condition.setMetricNamesNotCondition(true);
- condition.appendUuidClause(sb);
- Assert.assertEquals(sb.toString(), "(UUID NOT LIKE ?)");
- Assert.assertEquals(uuids.size(), condition.getUuids().size());
-
- //metric NOT LIKE host LIKE
- uuids.clear();
- uuids.add(new byte[16]);
- uuids.add(new byte[4]);
- condition = new DefaultCondition(uuids,null,null,null,null,null,null,null,null,true);
- sb = new StringBuilder();
- condition.setMetricNamesNotCondition(true);
- condition.appendUuidClause(sb);
- Assert.assertEquals(sb.toString(), "(UUID NOT LIKE ? AND UUID LIKE ?)");
- Assert.assertEquals(uuids.size(), condition.getUuids().size());
- Assert.assertTrue(new String(condition.getUuids().get(0)).endsWith("%"));
- Assert.assertTrue(new String(condition.getUuids().get(1)).startsWith("%"));
-
- //metric LIKE host NOT LIKE
- uuids.clear();
- uuids.add(new byte[16]);
- uuids.add(new byte[4]);
- condition = new DefaultCondition(uuids,null,null,null,null,null,null,null,null,true);
- sb = new StringBuilder();
- condition.setHostnamesNotCondition(true);
- condition.appendUuidClause(sb);
- Assert.assertEquals(sb.toString(), "(UUID LIKE ? AND UUID NOT LIKE ?)");
- Assert.assertEquals(uuids.size(), condition.getUuids().size());
- Assert.assertTrue(new String(condition.getUuids().get(0)).endsWith("%"));
- Assert.assertTrue(new String(condition.getUuids().get(1)).startsWith("%"));
-
- //metric LIKE or LIKE host LIKE
- uuids.clear();
- uuids.add(new byte[4]);
- uuids.add(new byte[16]);
- uuids.add(new byte[16]);
- condition = new DefaultCondition(uuids,null,null,null,null,null,null,null,null,true);
- sb = new StringBuilder();
- condition.appendUuidClause(sb);
- Assert.assertEquals(sb.toString(), "((UUID LIKE ? OR UUID LIKE ?) AND UUID LIKE ?)");
- Assert.assertEquals(uuids.size(), condition.getUuids().size());
- Assert.assertTrue(new String(condition.getUuids().get(0)).endsWith("%"));
- Assert.assertTrue(new String(condition.getUuids().get(1)).endsWith("%"));
- Assert.assertTrue(new String(condition.getUuids().get(2)).startsWith("%"));
-
- //UUID in metric LIKE or LIKE host LIKE
- uuids.clear();
- uuids.add(new byte[16]);
- uuids.add(new byte[16]);
- uuids.add(new byte[20]);
- uuids.add(new byte[4]);
- condition = new DefaultCondition(uuids,null,null,null,null,null,null,null,null,true);
- sb = new StringBuilder();
- condition.appendUuidClause(sb);
- Assert.assertEquals(sb.toString(), "(UUID IN (?) AND (UUID LIKE ? OR UUID LIKE ?) AND UUID LIKE ?)");
- Assert.assertEquals(uuids.size(), condition.getUuids().size());
- Assert.assertTrue(new String(condition.getUuids().get(1)).endsWith("%"));
- Assert.assertTrue(new String(condition.getUuids().get(2)).endsWith("%"));
- Assert.assertTrue(new String(condition.getUuids().get(3)).startsWith("%"));
-
- //metric LIKE host LIKE or LIKE
- uuids.clear();
- uuids.add(new byte[16]);
- uuids.add(new byte[4]);
- uuids.add(new byte[4]);
- condition = new DefaultCondition(uuids,null,null,null,null,null,null,null,null,true);
- sb = new StringBuilder();
- condition.appendUuidClause(sb);
- Assert.assertEquals(sb.toString(), "(UUID LIKE ? AND (UUID LIKE ? OR UUID LIKE ?))");
- Assert.assertEquals(uuids.size(), condition.getUuids().size());
- Assert.assertTrue(new String(condition.getUuids().get(0)).endsWith("%"));
- Assert.assertTrue(new String(condition.getUuids().get(1)).startsWith("%"));
- Assert.assertTrue(new String(condition.getUuids().get(2)).startsWith("%"));
-
- //UUID NOT IN metric LIKE host LIKE
- uuids.clear();
- uuids.add(new byte[20]);
- uuids.add(new byte[16]);
- uuids.add(new byte[4]);
- condition = new DefaultCondition(uuids,null,null,null,null,null,null,null,null,true);
- sb = new StringBuilder();
- condition.setUuidNotCondition(true);
- condition.appendUuidClause(sb);
- Assert.assertEquals(sb.toString(), "(UUID NOT IN (?) AND UUID LIKE ? AND UUID LIKE ?)");
- Assert.assertEquals(uuids.size(), condition.getUuids().size());
- Assert.assertTrue(new String(condition.getUuids().get(1)).endsWith("%"));
- Assert.assertTrue(new String(condition.getUuids().get(2)).startsWith("%"));
- }
-}
-
[18/28] ambari git commit: AMBARI-22740 : Fix integration test for
HBase in branch-3.0-ams due to UUID changes. (avijayan)
Posted by av...@apache.org.
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/MetricsLoadSimulator.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/MetricsLoadSimulator.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/MetricsLoadSimulator.java
deleted file mode 100644
index e85c7a5..0000000
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/MetricsLoadSimulator.java
+++ /dev/null
@@ -1,139 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics
- .loadsimulator;
-
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.Map;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Sample Usage:
- * <pre>
- * $ java -cp "dependency/*":LoadSimulator-1.0-SNAPSHOT.jar \
- * org.apache.ambari.metrics.MetricsLoadSimulator \
- * -h "bartosz.laptop" -n 2 -m "162.216.148.45" -c 10000 -s 30000</pre>
- */
-public class MetricsLoadSimulator {
- private final static Logger LOG = LoggerFactory.getLogger(MetricsLoadSimulator
- .class);
-
- public static void main(String[] args) throws IOException, InterruptedException {
- Map<String, String> mapArgs = parseArgs(args);
-
- LoadRunner loadRunner = new LoadRunner(
- mapArgs.get("hostName"),
- Integer.valueOf(mapArgs.get("numberOfHosts")),
- mapArgs.get("metricsHostName"),
- mapArgs.get("minHostIndex") == null ? 0 : Integer.valueOf(mapArgs.get("minHostIndex")),
- Integer.valueOf(mapArgs.get("collectInterval")),
- Integer.valueOf(mapArgs.get("sendInterval")),
- Boolean.valueOf(mapArgs.get("master"))
- );
-
- loadRunner.start();
- }
-
- /*
- Another entry point to the test to be called from ./jmetertest/AMSJMeterLoadTest.java
- */
- public static void startTest(Map<String,String> mapArgs) {
-
- LoadRunner loadRunner = new LoadRunner(
- mapArgs.get("hostName"),
- Integer.valueOf(mapArgs.get("numberOfHosts")),
- mapArgs.get("metricsHostName"),
- mapArgs.get("minHostIndex") == null ? 0 : Integer.valueOf(mapArgs.get("minHostIndex")),
- Integer.valueOf(mapArgs.get("collectInterval")),
- Integer.valueOf(mapArgs.get("sendInterval")),
- Boolean.valueOf(mapArgs.get("master"))
- );
-
- loadRunner.start();
- }
-
- private static Map<String, String> parseArgs(String[] args) {
- Map<String, String> mapProps = new HashMap<String, String>();
- mapProps.put("hostName", "host");
- mapProps.put("numberOfHosts", "20");
- mapProps.put("trafficType", "burst");
- mapProps.put("metricsHostName", "localhost");
- mapProps.put("collectInterval", "10000");
- mapProps.put("sendInterval", "60000");
-
- if (args.length == 0) {
- printUsage();
- throw new RuntimeException("Unexpected argument, See usage message.");
- } else {
- for (int i = 0; i < args.length; i += 2) {
- String arg = args[i];
- if (arg.equals("-h")) {
- mapProps.put("hostName", args[i + 1]);
- } else if (arg.equals("-n")) {
- mapProps.put("numberOfHosts", args[i + 1]);
- } else if (arg.equals("-t")) {
- mapProps.put("trafficType", args[i + 1]);
- } else if (arg.equals("-m")) {
- mapProps.put("metricsHostName", args[i + 1]);
- } else if (arg.equals("-c")) {
- mapProps.put("collectInterval", args[i + 1]);
- } else if (arg.equals("-s")) {
- mapProps.put("sendInterval", args[i + 1]);
- } else if (arg.equals("-M")) {
- mapProps.put("master", args[i + 1]);
- } else if (arg.equals("-d")) {
- // a dummy switch - it says that we agree with defaults
- } else {
- printUsage();
- throw new RuntimeException("Unexpected argument, See usage message.");
- }
- }
- }
-
- LOG.info("Recognized options: baseHostName={} hosts#={} trafficMode={} " +
- "metricsHostName={} collectIntervalMillis={} sendIntervalMillis={} " +
- "simulateMaster={}",
- mapProps.get("hostName"),
- Integer.valueOf(mapProps.get("numberOfHosts")),
- mapProps.get("trafficType"),
- mapProps.get("metricsHostName"),
- Integer.valueOf(mapProps.get("collectInterval")),
- Integer.valueOf(mapProps.get("sendInterval")),
- Boolean.valueOf(mapProps.get("master"))
- );
-
- return mapProps;
- }
-
- public static void printUsage() {
- System.err.println("Usage: java MetricsLoadSimulator [OPTIONS]");
- System.err.println("Options: ");
- System.err.println("[-h hostName] [-n numberOfHosts] "
- + "[-t trafficMode {burst, staggered}] [-m metricsHostName] "
- + "[-c collectIntervalMillis {10 sec}] [-s sendIntervalMillis {60 sec}]"
- + "[-M simulateMaster {true, false}] ");
- System.err.println();
- System.err.println("When you select a master, then one simulated host will play");
- System.err.println("a role of a master, and the rest will be slaves. Otherwise");
- System.err.println("all simulation threads (single thread is for single host)");
- System.err.println("will be slave hosts");
- }
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/MetricsSenderWorker.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/MetricsSenderWorker.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/MetricsSenderWorker.java
deleted file mode 100644
index 71f2bc5..0000000
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/MetricsSenderWorker.java
+++ /dev/null
@@ -1,60 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.loadsimulator;
-
-
-import java.io.IOException;
-import java.util.concurrent.Callable;
-
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.loadsimulator.data.AppMetrics;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.loadsimulator.data.HostMetricsGenerator;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.loadsimulator.net.MetricsSender;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.loadsimulator.net.RestMetricsSender;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.loadsimulator.util.Json;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- */
-public class MetricsSenderWorker implements Callable<String> {
- private final static Logger LOG = LoggerFactory.getLogger(RestMetricsSender.class);
-
- MetricsSender sender;
- HostMetricsGenerator hmg;
-
- public MetricsSenderWorker(MetricsSender sender, HostMetricsGenerator metricsGenerator) {
- this.sender = sender;
- hmg = metricsGenerator;
- }
-
- @Override
- public String call() throws Exception {
- AppMetrics hostMetrics = hmg.createMetrics();
-
- try {
- String request = new Json().serialize(hostMetrics); //inject?
- String response = sender.pushMetrics(request);
-
- return response;
- } catch (IOException e) {
- LOG.error("Error while pushing metrics: ", e);
- throw e;
- }
-
- }
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/data/AppID.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/data/AppID.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/data/AppID.java
deleted file mode 100644
index 56c3571..0000000
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/data/AppID.java
+++ /dev/null
@@ -1,46 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics
- .loadsimulator.data;
-
-public enum AppID {
- HOST("HOST"),
- NAMENODE("namenode"),
- RESOURCEMANAGER("resourcemanager"),
- DATANODE("datanode"),
- NODEMANAGER("nodemanager"),
- MASTER_HBASE("hbase"),
- SLAVE_HBASE("hbase"),
- NIMBUS("nimbus"),
- HIVEMETASTORE("hivemetastore"),
- HIVESERVER2("hiveserver2"),
- KAFKA_BROKER("kafka_broker");
-
- public static final AppID[] MASTER_APPS = {HOST, NAMENODE, RESOURCEMANAGER, MASTER_HBASE, KAFKA_BROKER, NIMBUS, HIVEMETASTORE, HIVESERVER2};
- public static final AppID[] SLAVE_APPS = {HOST, DATANODE, NODEMANAGER, SLAVE_HBASE};
-
- private String id;
-
- private AppID(String id) {
- this.id = id;
- }
-
- public String getId() {
- return id;
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/data/AppMetrics.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/data/AppMetrics.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/data/AppMetrics.java
deleted file mode 100644
index d9cec2b..0000000
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/data/AppMetrics.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics
- .loadsimulator.data;
-
-import java.util.ArrayList;
-import java.util.Collection;
-
-/**
- * AppMetrics is a class that helps to create properly initialized metrics for
- * current app. It also holds the
- * metrics and can be serialized to json.
- */
-public class AppMetrics {
-
- private final Collection<Metric> metrics = new ArrayList<Metric>();
- private final transient ApplicationInstance applicationId;
- private final transient long startTime;
-
- public AppMetrics(ApplicationInstance applicationId, long startTime) {
- this.applicationId = applicationId;
- this.startTime = startTime;
- }
-
- public Metric createMetric(String metricName) {
- return new Metric(applicationId, metricName, startTime);
- }
-
- public void addMetric(Metric metric) {
- metrics.add(metric);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/data/ApplicationInstance.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/data/ApplicationInstance.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/data/ApplicationInstance.java
deleted file mode 100644
index d99ecc9..0000000
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/data/ApplicationInstance.java
+++ /dev/null
@@ -1,59 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics
- .loadsimulator.data;
-
-/**
- * AppId is a helper class that encapsulates the common part of metrics message.
- * It contains hostName, appId and instanceId. It is immutable,
- * and it can not hold null values.
- */
-public final class ApplicationInstance {
-
- private final transient String hostName;
- private final transient AppID appId;
- private final transient String instanceId;
-
- /**
- * @param hostname
- * @param appId
- * @param instanceId
- */
- public ApplicationInstance(String hostname, AppID appId, String instanceId) {
- if (hostname == null || appId == null || instanceId == null)
- throw new IllegalArgumentException("ApplicationInstance can not be " +
- "instantiated with null values");
-
- this.hostName = hostname;
- this.appId = appId;
- this.instanceId = instanceId;
- }
-
- public String getInstanceId() {
- return instanceId;
- }
-
- public AppID getAppId() {
- return appId;
- }
-
- public String getHostName() {
- return hostName;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/data/HostMetricsGenerator.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/data/HostMetricsGenerator.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/data/HostMetricsGenerator.java
deleted file mode 100644
index f628f2c..0000000
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/data/HostMetricsGenerator.java
+++ /dev/null
@@ -1,61 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.loadsimulator.data;
-
-
-import java.util.HashMap;
-import java.util.Map;
-
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.loadsimulator.util.RandomMetricsProvider;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.loadsimulator.util.TimeStampProvider;
-
-/**
- */
-public class HostMetricsGenerator {
-
- private Map<String, RandomMetricsProvider> metricDataProviders = new HashMap<String, RandomMetricsProvider>();
- private final TimeStampProvider tsp;
- private final ApplicationInstance id;
-
- public HostMetricsGenerator(ApplicationInstance id,
- TimeStampProvider timeStamps,
- Map<String, RandomMetricsProvider> metricDataProviders) {
- this.id = id;
- this.tsp = timeStamps;
- this.metricDataProviders = metricDataProviders;
- }
-
- public AppMetrics createMetrics() {
- long[] timestamps = tsp.timestampsForNextInterval();
- AppMetrics appMetrics = new AppMetrics(id, timestamps[0]);
-
- for (Map.Entry<String, RandomMetricsProvider> entry : metricDataProviders.entrySet()) {
- String metricName = entry.getKey();
- RandomMetricsProvider metricData = entry.getValue();
-
- Metric metric = appMetrics.createMetric(metricName);
- for (long timestamp : timestamps) {
- metric.putMetric(timestamp, String.valueOf(metricData.next()));
- }
- appMetrics.addMetric(metric);
- }
-
- return appMetrics;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/data/Metric.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/data/Metric.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/data/Metric.java
deleted file mode 100644
index f274263..0000000
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/data/Metric.java
+++ /dev/null
@@ -1,71 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.loadsimulator.data;
-
-import java.util.LinkedHashMap;
-import java.util.Map;
-
-public class Metric {
-
- private String instanceid;
- private String hostname;
- private Map<String, String> metrics = new LinkedHashMap<String, String>();
- private String starttime;
- private String appid;
- private String metricname;
-
- // i don't like this ctor, but it has to be public for json deserialization
- public Metric() {
- }
-
- public Metric(ApplicationInstance app, String metricName, long startTime) {
- this.hostname = app.getHostName();
- this.appid = app.getAppId().getId();
- this.instanceid = app.getInstanceId();
- this.metricname = metricName;
- this.starttime = Long.toString(startTime);
- }
-
- public void putMetric(long timestamp, String value) {
- metrics.put(Long.toString(timestamp), value);
- }
-
- public String getInstanceid() {
- return instanceid;
- }
-
- public String getHostname() {
- return hostname;
- }
-
- public Map<String, String> getMetrics() {
- return metrics;
- }
-
- public String getStarttime() {
- return starttime;
- }
-
- public String getAppid() {
- return appid;
- }
-
- public String getMetricname() {
- return metricname;
- }
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/data/MetricsGeneratorConfigurer.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/data/MetricsGeneratorConfigurer.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/data/MetricsGeneratorConfigurer.java
deleted file mode 100644
index b315541..0000000
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/data/MetricsGeneratorConfigurer.java
+++ /dev/null
@@ -1,93 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.loadsimulator.data;
-
-
-import java.io.BufferedReader;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.util.HashMap;
-import java.util.Map;
-
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.loadsimulator.util.RandomMetricsProvider;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.loadsimulator.util.TimeStampProvider;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * MetricsGeneratorConfigurer is a factory that reads metrics definition from a file,
- * and returns an Single HostMetricsGenerator. Check createMetricsForHost method
- * for details.
- */
-public class MetricsGeneratorConfigurer {
-
- private final static Logger LOG = LoggerFactory.getLogger
- (MetricsGeneratorConfigurer.class);
-
- /**
- * Creates HostMetricsGenerator configured with metric names loaded from file.
- *
- * @param id ApplicationInstance descriptor, will be used to create
- * HostMetricsGenerator, cannot be null
- * @param timeStamps configured TimeStampProvider that can provide next
- * timestamp, cannot be null
- * @return HostMetricsGenerator with given ApplicationInstance id and configured
- * mapping of
- * metric names to data providers
- */
- public static HostMetricsGenerator createMetricsForHost(
- ApplicationInstance id,
- TimeStampProvider timeStamps) {
- return new HostMetricsGenerator(id, timeStamps, readMetrics(id.getAppId()));
- }
-
- private static Map<String, RandomMetricsProvider> readMetrics(AppID type) {
- InputStream input = null;
- Map<String, RandomMetricsProvider> metrics =
- new HashMap<String, RandomMetricsProvider>();
- String fileName = "metrics_def/" + type.toString() + ".dat";
-
- try {
- LOG.info("Loading " + fileName);
-
- input = MetricsGeneratorConfigurer.class.getClassLoader()
- .getResourceAsStream(fileName);
-
- BufferedReader reader = new BufferedReader(new InputStreamReader(input));
-
- String line;
- while ((line = reader.readLine()) != null) {
- metrics.put(line.trim(), new RandomMetricsProvider(100, 200));
- }
-
- } catch (IOException e) {
- LOG.error("Cannot read file " + fileName + " for appID " + type.toString(), e);
- } finally {
- if (input != null) {
- try {
- input.close();
- } catch (IOException ex) {
- // intentionally left blank, here we cannot do anything
- }
- }
- }
-
- return metrics;
- }
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/net/MetricsSender.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/net/MetricsSender.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/net/MetricsSender.java
deleted file mode 100644
index 35c0fc3..0000000
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/net/MetricsSender.java
+++ /dev/null
@@ -1,31 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.loadsimulator.net;
-
-/**
- * MetricSender should provides a simple way of pushing metrics to some service.
- */
-public interface MetricsSender {
- /**
- * Push metrics to the metric service (e.g. a metrics storage system).
- *
- * @param payload the payload to be sent to metrics service
- * @return response message either acknowledgement or error
- */
- String pushMetrics(String payload);
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/net/RestMetricsSender.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/net/RestMetricsSender.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/net/RestMetricsSender.java
deleted file mode 100644
index 8eb3fec..0000000
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/net/RestMetricsSender.java
+++ /dev/null
@@ -1,92 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.loadsimulator.net;
-
-import java.io.IOException;
-import java.util.concurrent.TimeUnit;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.google.common.base.Stopwatch;
-
-/**
- * Implements MetricsSender and provides a way of pushing metrics to application metrics history service using REST
- * endpoint.
- */
-public class RestMetricsSender implements MetricsSender {
- private final static Logger LOG = LoggerFactory.getLogger(RestMetricsSender.class);
-
- private final static String COLLECTOR_URL = "http://%s/ws/v1/timeline/metrics";
- private final String collectorServiceAddress;
-
- /**
- * Creates unconnected RestMetricsSender with endpoint configured as
- * http://${metricsHost}:6188/ws/v1/timeline/metrics,
- * where ${metricsHost} is specified by metricHost param.
- *
- * @param metricsHost the hostname that will be used to access application metrics history service.
- */
- public RestMetricsSender(String metricsHost) {
- collectorServiceAddress = String.format(COLLECTOR_URL, metricsHost);
- }
-
- /**
- * Push metrics to the REST endpoint. Connection is always open and closed on every call.
- *
- * @param payload the payload with metrics to be sent to metrics service
- * @return response message either acknowledgement or error, empty on exception
- */
- @Override
- public String pushMetrics(String payload) {
- String responseString = "";
- UrlService svc = null;
- Stopwatch timer = new Stopwatch().start();
-
- try {
- LOG.info("server: {}", collectorServiceAddress);
-
- svc = getConnectedUrlService();
- responseString = svc.send(payload);
-
- timer.stop();
- LOG.info("http response time: " + timer.elapsed(TimeUnit.MILLISECONDS)
- + " ms");
-
- if (responseString.length() > 0) {
- LOG.debug("POST response from server: " + responseString);
- }
- } catch (IOException e) {
- LOG.error("", e);
- } finally {
- if (svc != null) {
- svc.disconnect();
- }
- }
-
- return responseString;
- }
-
- /**
- * Relaxed to protected for testing.
- */
- protected UrlService getConnectedUrlService() throws IOException {
- return UrlService.newConnection(collectorServiceAddress);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/net/StdOutMetricsSender.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/net/StdOutMetricsSender.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/net/StdOutMetricsSender.java
deleted file mode 100644
index aeb4ca8..0000000
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/net/StdOutMetricsSender.java
+++ /dev/null
@@ -1,56 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.loadsimulator.net;
-
-import java.io.PrintStream;
-
-/**
- * StdOutMetricsSender dumps metrics to defined PrintStream out. It is useful for testing.
- */
-public class StdOutMetricsSender implements MetricsSender {
- public final PrintStream out;
- private String metricsHostName;
-
- /**
- * Creates new StdOutMetricsSender with specified hostname (only used in messages) and sends output to System.out
- *
- * @param metricsHostName a name used in printed messages
- */
- public StdOutMetricsSender(String metricsHostName) {
- this(metricsHostName, System.out);
- }
-
- /**
- * Creates new StdOutMetricsSender with specified hostname (only used in messages) and PrintStream which is used as
- * an output.
- *
- * @param metricsHostName a name used in printed messages
- * @param out PrintStream that the Sender will write to, can be System.out
- */
- public StdOutMetricsSender(String metricsHostName, PrintStream out) {
- this.metricsHostName = metricsHostName;
- this.out = out;
- }
-
- @Override
- public String pushMetrics(String payload) {
- out.println("Sending to " + metricsHostName + ": " + payload);
-
- return "OK";
- }
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/net/UrlService.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/net/UrlService.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/net/UrlService.java
deleted file mode 100644
index 7402438..0000000
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/net/UrlService.java
+++ /dev/null
@@ -1,101 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics
- .loadsimulator.net;
-
-import java.io.BufferedReader;
-import java.io.IOException;
-import java.io.InputStreamReader;
-import java.io.OutputStreamWriter;
-import java.net.HttpURLConnection;
-import java.net.URL;
-
-public class UrlService {
-
- public static final int CONNECT_TIMEOUT = 20000;
- public static final int READ_TIMEOUT = 20000;
- private final String address;
- private HttpURLConnection conn;
-
- private UrlService(String address) {
- this.address = address;
- }
-
- /**
- * Returns a new UrlService connected to specified address.
- *
- * @param address
- * @return
- * @throws IOException
- */
- public static UrlService newConnection(String address) throws IOException {
- UrlService svc = new UrlService(address);
- svc.connect();
-
- return svc;
- }
-
- public HttpURLConnection connect() throws IOException {
- URL url = new URL(address);
- conn = (HttpURLConnection) url.openConnection();
-
- //TODO: make timeouts configurable
- conn.setConnectTimeout(CONNECT_TIMEOUT);
- conn.setReadTimeout(READ_TIMEOUT);
- conn.setDoInput(true);
- conn.setDoOutput(true);
- conn.setRequestMethod("POST");
- conn.setRequestProperty("Content-Type", "application/json");
- conn.setRequestProperty("Accept", "*/*");
-
- return conn;
- }
-
- public String send(String payload) throws IOException {
- if (conn == null)
- throw new IllegalStateException("Cannot use unconnected UrlService");
- write(payload);
-
- return read();
- }
-
- private String read() throws IOException {
- StringBuilder response = new StringBuilder();
-
- BufferedReader br = new BufferedReader(new InputStreamReader(
- conn.getInputStream()));
- String line = null;
- while ((line = br.readLine()) != null) {
- response.append(line);
- }
- br.close();
-
- return response.toString();
- }
-
- private void write(String payload) throws IOException {
- OutputStreamWriter writer = new OutputStreamWriter(conn.getOutputStream(),
- "UTF-8");
- writer.write(payload);
- writer.close();
- }
-
- public void disconnect() {
- conn.disconnect();
- }
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/util/Json.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/util/Json.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/util/Json.java
deleted file mode 100644
index 982f48c..0000000
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/util/Json.java
+++ /dev/null
@@ -1,63 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics
- .loadsimulator.util;
-
-import java.io.IOException;
-
-import org.codehaus.jackson.annotate.JsonAutoDetect;
-import org.codehaus.jackson.annotate.JsonMethod;
-import org.codehaus.jackson.map.ObjectMapper;
-import org.codehaus.jackson.map.SerializationConfig;
-
-/**
- * Small wrapper that configures the ObjectMapper with some defaults.
- */
-public class Json {
- private ObjectMapper myObjectMapper;
-
- /**
- * Creates default Json ObjectMapper that maps fields.
- */
- public Json() {
- this(false);
- }
-
- /**
- * Creates a Json ObjectMapper that maps fields and optionally pretty prints the
- * serialized objects.
- *
- * @param pretty a flag - if true the output will be pretty printed.
- */
- public Json(boolean pretty) {
- myObjectMapper = new ObjectMapper();
- myObjectMapper.setVisibility(JsonMethod.FIELD, JsonAutoDetect.Visibility.ANY);
- if (pretty) {
- myObjectMapper.configure(SerializationConfig.Feature.INDENT_OUTPUT, true);
- }
- }
-
- public String serialize(Object o) throws IOException {
- return myObjectMapper.writeValueAsString(o);
- }
-
- public <T> T deserialize(String content, Class<T> paramClass) throws IOException {
- return myObjectMapper.readValue(content, paramClass);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/util/RandomMetricsProvider.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/util/RandomMetricsProvider.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/util/RandomMetricsProvider.java
deleted file mode 100644
index 7910711..0000000
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/util/RandomMetricsProvider.java
+++ /dev/null
@@ -1,40 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics
- .loadsimulator.util;
-
-import java.util.Random;
-
-/**
- */
-public class RandomMetricsProvider {
-
- private double min;
- private double max;
- private Random rnd;
-
- public RandomMetricsProvider(double min, double max) {
- this.min = min;
- this.max = max;
- this.rnd = new Random();
- }
-
- public double next() {
- return rnd.nextDouble() * (max - min) + min;
- }
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/util/TimeStampProvider.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/util/TimeStampProvider.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/util/TimeStampProvider.java
deleted file mode 100644
index ad7ec86..0000000
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/util/TimeStampProvider.java
+++ /dev/null
@@ -1,52 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics
- .loadsimulator.util;
-
-/**
- */
-public class TimeStampProvider {
- private int timeStep;
- private long currentTime;
- private int sendInterval;
-
- public TimeStampProvider(long startTime, int timeStep, int sendInterval) {
- this.timeStep = timeStep;
- this.currentTime = startTime - timeStep;
- this.sendInterval = sendInterval;
- }
-
- public long next() {
- return currentTime += timeStep;
- }
-
- public long[] timestampsForNextInterval() {
- return timestampsForInterval(sendInterval);
- }
-
- private long[] timestampsForInterval(int sendInterval) {
- int steps = sendInterval / timeStep;
- long[] timestamps = new long[steps];
-
- for (int i = 0; i < timestamps.length; i++) {
- timestamps[i] = next();
- }
-
- return timestamps;
- }
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/HBaseTimelineMetricsService.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/HBaseTimelineMetricsService.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/HBaseTimelineMetricsService.java
deleted file mode 100644
index c148a43..0000000
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/HBaseTimelineMetricsService.java
+++ /dev/null
@@ -1,594 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline;
-
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.DEFAULT_TOPN_HOSTS_LIMIT;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.USE_GROUPBY_AGGREGATOR_QUERIES;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.availability.AggregationTaskRunner.ACTUAL_AGGREGATOR_NAMES;
-
-import java.io.IOException;
-import java.net.MalformedURLException;
-import java.net.URISyntaxException;
-import java.net.UnknownHostException;
-import java.sql.SQLException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.Executors;
-import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.ThreadFactory;
-import java.util.concurrent.TimeUnit;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-import org.apache.commons.collections.MapUtils;
-import org.apache.commons.lang.StringUtils;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.metrics2.sink.timeline.AggregationResult;
-import org.apache.hadoop.metrics2.sink.timeline.ContainerMetric;
-import org.apache.hadoop.metrics2.sink.timeline.MetricHostAggregate;
-import org.apache.hadoop.metrics2.sink.timeline.Precision;
-import org.apache.hadoop.metrics2.sink.timeline.TimelineMetric;
-import org.apache.hadoop.metrics2.sink.timeline.TimelineMetricMetadata;
-import org.apache.hadoop.metrics2.sink.timeline.TimelineMetricWithAggregatedValues;
-import org.apache.hadoop.metrics2.sink.timeline.TimelineMetrics;
-import org.apache.hadoop.metrics2.sink.timeline.TopNConfig;
-import org.apache.hadoop.service.AbstractService;
-import org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.aggregators.Function;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.aggregators.TimelineMetricAggregator;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.aggregators.TimelineMetricAggregatorFactory;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.availability.AggregationTaskRunner.AGGREGATOR_NAME;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.availability.MetricCollectorHAController;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.discovery.TimelineMetricHostMetadata;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.discovery.TimelineMetricMetadataKey;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.discovery.TimelineMetricMetadataManager;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.function.SeriesAggregateFunction;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.function.TimelineMetricsSeriesAggregateFunction;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.function.TimelineMetricsSeriesAggregateFunctionFactory;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query.Condition;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query.ConditionBuilder;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query.PhoenixTransactSQL;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query.TopNCondition;
-
-import com.google.common.collect.ArrayListMultimap;
-import com.google.common.collect.Multimap;
-
-public class HBaseTimelineMetricsService extends AbstractService implements TimelineMetricStore {
-
- static final Log LOG = LogFactory.getLog(HBaseTimelineMetricsService.class);
- private final TimelineMetricConfiguration configuration;
- private TimelineMetricDistributedCache cache;
- private PhoenixHBaseAccessor hBaseAccessor;
- private static volatile boolean isInitialized = false;
- private final ScheduledExecutorService watchdogExecutorService = Executors.newSingleThreadScheduledExecutor();
- private final Map<AGGREGATOR_NAME, ScheduledExecutorService> scheduledExecutors = new HashMap<>();
- private final ConcurrentHashMap<String, Long> postedAggregatedMap = new ConcurrentHashMap<>();
- private TimelineMetricMetadataManager metricMetadataManager;
- private Integer defaultTopNHostsLimit;
- private MetricCollectorHAController haController;
- private boolean containerMetricsDisabled = false;
-
- /**
- * Construct the service.
- *
- */
- public HBaseTimelineMetricsService(TimelineMetricConfiguration configuration) {
- super(HBaseTimelineMetricsService.class.getName());
- this.configuration = configuration;
- }
-
- @Override
- protected void serviceInit(Configuration conf) throws Exception {
- super.serviceInit(conf);
- initializeSubsystem();
- }
-
- private TimelineMetricDistributedCache startCacheNode() throws MalformedURLException, URISyntaxException {
- //TODO make configurable
- return new TimelineMetricsIgniteCache();
- }
-
-
- private synchronized void initializeSubsystem() {
- if (!isInitialized) {
- hBaseAccessor = new PhoenixHBaseAccessor(null);
- // Initialize schema
- hBaseAccessor.initMetricSchema();
- // Initialize metadata from store
- try {
- metricMetadataManager = new TimelineMetricMetadataManager(hBaseAccessor);
- } catch (MalformedURLException | URISyntaxException e) {
- throw new ExceptionInInitializerError("Unable to initialize metadata manager");
- }
- metricMetadataManager.initializeMetadata();
- // Initialize policies before TTL update
- hBaseAccessor.initPoliciesAndTTL();
- // Start HA service
- // Start the controller
- if (!configuration.isDistributedCollectorModeDisabled()) {
- haController = new MetricCollectorHAController(configuration);
- try {
- haController.initializeHAController();
- } catch (Exception e) {
- LOG.error(e);
- throw new MetricsSystemInitializationException("Unable to " +
- "initialize HA controller", e);
- }
- } else {
- LOG.info("Distributed collector mode disabled");
- }
-
- //Initialize whitelisting & blacklisting if needed
- TimelineMetricsFilter.initializeMetricFilter(configuration);
-
- Configuration metricsConf = null;
- try {
- metricsConf = configuration.getMetricsConf();
- } catch (Exception e) {
- throw new ExceptionInInitializerError("Cannot initialize configuration.");
- }
-
- if (configuration.isCollectorInMemoryAggregationEnabled()) {
- try {
- cache = startCacheNode();
- } catch (Exception e) {
- throw new MetricsSystemInitializationException("Unable to " +
- "start cache node", e);
- }
- }
-
- defaultTopNHostsLimit = Integer.parseInt(metricsConf.get(DEFAULT_TOPN_HOSTS_LIMIT, "20"));
- if (Boolean.parseBoolean(metricsConf.get(USE_GROUPBY_AGGREGATOR_QUERIES, "true"))) {
- LOG.info("Using group by aggregators for aggregating host and cluster metrics.");
- }
-
- // Start the cluster aggregator second
- TimelineMetricAggregator secondClusterAggregator =
- TimelineMetricAggregatorFactory.createTimelineClusterAggregatorSecond(
- hBaseAccessor, metricsConf, metricMetadataManager, haController, cache);
- scheduleAggregatorThread(secondClusterAggregator);
-
- // Start the minute cluster aggregator
- TimelineMetricAggregator minuteClusterAggregator =
- TimelineMetricAggregatorFactory.createTimelineClusterAggregatorMinute(
- hBaseAccessor, metricsConf, metricMetadataManager, haController);
- scheduleAggregatorThread(minuteClusterAggregator);
-
- // Start the hourly cluster aggregator
- TimelineMetricAggregator hourlyClusterAggregator =
- TimelineMetricAggregatorFactory.createTimelineClusterAggregatorHourly(
- hBaseAccessor, metricsConf, metricMetadataManager, haController);
- scheduleAggregatorThread(hourlyClusterAggregator);
-
- // Start the daily cluster aggregator
- TimelineMetricAggregator dailyClusterAggregator =
- TimelineMetricAggregatorFactory.createTimelineClusterAggregatorDaily(
- hBaseAccessor, metricsConf, metricMetadataManager, haController);
- scheduleAggregatorThread(dailyClusterAggregator);
-
- // Start the minute host aggregator
- if (configuration.isHostInMemoryAggregationEnabled()) {
- LOG.info("timeline.metrics.host.inmemory.aggregation is set to True, switching to filtering host minute aggregation on collector");
- TimelineMetricAggregator minuteHostAggregator =
- TimelineMetricAggregatorFactory.createFilteringTimelineMetricAggregatorMinute(
- hBaseAccessor, metricsConf, metricMetadataManager, haController, postedAggregatedMap);
- scheduleAggregatorThread(minuteHostAggregator);
- } else {
- TimelineMetricAggregator minuteHostAggregator =
- TimelineMetricAggregatorFactory.createTimelineMetricAggregatorMinute(
- hBaseAccessor, metricsConf, metricMetadataManager, haController);
- scheduleAggregatorThread(minuteHostAggregator);
- }
-
- // Start the hourly host aggregator
- TimelineMetricAggregator hourlyHostAggregator =
- TimelineMetricAggregatorFactory.createTimelineMetricAggregatorHourly(
- hBaseAccessor, metricsConf, metricMetadataManager, haController);
- scheduleAggregatorThread(hourlyHostAggregator);
-
- // Start the daily host aggregator
- TimelineMetricAggregator dailyHostAggregator =
- TimelineMetricAggregatorFactory.createTimelineMetricAggregatorDaily(
- hBaseAccessor, metricsConf, metricMetadataManager, haController);
- scheduleAggregatorThread(dailyHostAggregator);
-
- if (!configuration.isTimelineMetricsServiceWatcherDisabled()) {
- int initDelay = configuration.getTimelineMetricsServiceWatcherInitDelay();
- int delay = configuration.getTimelineMetricsServiceWatcherDelay();
- // Start the watchdog
- watchdogExecutorService.scheduleWithFixedDelay(
- new TimelineMetricStoreWatcher(this, configuration),
- initDelay, delay, TimeUnit.SECONDS);
- LOG.info("Started watchdog for timeline metrics store with initial " +
- "delay = " + initDelay + ", delay = " + delay);
- }
- containerMetricsDisabled = configuration.isContainerMetricsDisabled();
- isInitialized = true;
- }
-
- }
-
- @Override
- protected void serviceStop() throws Exception {
- super.serviceStop();
- }
-
- @Override
- public TimelineMetrics getTimelineMetrics(List<String> metricNames,
- List<String> hostnames, String applicationId, String instanceId,
- Long startTime, Long endTime, Precision precision, Integer limit,
- boolean groupedByHosts, TopNConfig topNConfig, String seriesAggregateFunction) throws SQLException, IOException {
-
- if (metricNames == null || metricNames.isEmpty()) {
- throw new IllegalArgumentException("No metric name filter specified.");
- }
- if ((startTime == null && endTime != null)
- || (startTime != null && endTime == null)) {
- throw new IllegalArgumentException("Open ended query not supported ");
- }
- if (limit != null && limit > PhoenixHBaseAccessor.RESULTSET_LIMIT){
- throw new IllegalArgumentException("Limit too big");
- }
-
- TimelineMetricsSeriesAggregateFunction seriesAggrFunctionInstance = null;
- if (!StringUtils.isEmpty(seriesAggregateFunction)) {
- SeriesAggregateFunction func = SeriesAggregateFunction.getFunction(seriesAggregateFunction);
- seriesAggrFunctionInstance = TimelineMetricsSeriesAggregateFunctionFactory.newInstance(func);
- }
-
- Multimap<String, List<Function>> metricFunctions =
- parseMetricNamesToAggregationFunctions(metricNames);
-
- List<byte[]> uuids = metricMetadataManager.getUuids(metricFunctions.keySet(), hostnames, applicationId, instanceId);
-
- ConditionBuilder conditionBuilder = new ConditionBuilder(new ArrayList<String>(metricFunctions.keySet()))
- .hostnames(hostnames)
- .appId(applicationId)
- .instanceId(instanceId)
- .startTime(startTime)
- .endTime(endTime)
- .precision(precision)
- .limit(limit)
- .grouped(groupedByHosts)
- .uuid(uuids);
-
- if (topNConfig != null) {
- if (TopNCondition.isTopNHostCondition(metricNames, hostnames) ^ //Only 1 condition should be true.
- TopNCondition.isTopNMetricCondition(metricNames, hostnames)) {
- conditionBuilder.topN(topNConfig.getTopN());
- conditionBuilder.isBottomN(topNConfig.getIsBottomN());
- Function.ReadFunction readFunction = Function.ReadFunction.getFunction(topNConfig.getTopNFunction());
- Function function = new Function(readFunction, null);
- conditionBuilder.topNFunction(function);
- } else {
- LOG.info("Invalid Input for TopN query. Ignoring TopN Request.");
- }
- } else if (startTime != null && hostnames != null && hostnames.size() > defaultTopNHostsLimit) {
- // if (timeseries query AND hostnames passed AND size(hostnames) > limit)
- LOG.info("Requesting data for more than " + defaultTopNHostsLimit + " Hosts. " +
- "Defaulting to Top " + defaultTopNHostsLimit);
- conditionBuilder.topN(defaultTopNHostsLimit);
- conditionBuilder.isBottomN(false);
- }
-
- Condition condition = conditionBuilder.build();
-
- TimelineMetrics metrics;
-
- if (hostnames == null || hostnames.isEmpty()) {
- metrics = hBaseAccessor.getAggregateMetricRecords(condition, metricFunctions);
- } else {
- metrics = hBaseAccessor.getMetricRecords(condition, metricFunctions);
- }
-
- metrics = postProcessMetrics(metrics);
-
- if (metrics.getMetrics().size() == 0) {
- return metrics;
- }
-
- return seriesAggregateMetrics(seriesAggrFunctionInstance, metrics);
- }
-
- private TimelineMetrics postProcessMetrics(TimelineMetrics metrics) {
- List<TimelineMetric> metricsList = metrics.getMetrics();
-
- for (TimelineMetric metric : metricsList){
- String name = metric.getMetricName();
- if (name.contains("._rate")){
- updateValuesAsRate(metric.getMetricValues(), false);
- } else if (name.contains("._diff")) {
- updateValuesAsRate(metric.getMetricValues(), true);
- }
- }
-
- return metrics;
- }
-
- private TimelineMetrics seriesAggregateMetrics(TimelineMetricsSeriesAggregateFunction seriesAggrFuncInstance,
- TimelineMetrics metrics) {
- if (seriesAggrFuncInstance != null) {
- TimelineMetric appliedMetric = seriesAggrFuncInstance.apply(metrics);
- metrics.setMetrics(Collections.singletonList(appliedMetric));
- }
- return metrics;
- }
-
- static Map<Long, Double> updateValuesAsRate(Map<Long, Double> metricValues, boolean isDiff) {
- Long prevTime = null;
- Double prevVal = null;
- long step;
- Double diff;
-
- for(Iterator<Map.Entry<Long, Double>> it = metricValues.entrySet().iterator(); it.hasNext(); ) {
- Map.Entry<Long, Double> timeValueEntry = it.next();
- Long currTime = timeValueEntry.getKey();
- Double currVal = timeValueEntry.getValue();
-
- if (prevTime != null) {
- step = currTime - prevTime;
- diff = currVal - prevVal;
- if (diff < 0) {
- it.remove(); //Discard calculating rate when the metric counter has been reset.
- } else {
- Double rate = isDiff ? diff : (diff / TimeUnit.MILLISECONDS.toSeconds(step));
- timeValueEntry.setValue(rate);
- }
- } else {
- it.remove();
- }
-
- prevTime = currTime;
- prevVal = currVal;
- }
-
- return metricValues;
- }
-
- static Multimap<String, List<Function>> parseMetricNamesToAggregationFunctions(List<String> metricNames) {
- Multimap<String, List<Function>> metricsFunctions = ArrayListMultimap.create();
-
- for (String metricName : metricNames){
- Function function = Function.DEFAULT_VALUE_FUNCTION;
- String cleanMetricName = metricName;
-
- try {
- function = Function.fromMetricName(metricName);
- int functionStartIndex = metricName.indexOf("._");
- if (functionStartIndex > 0) {
- cleanMetricName = metricName.substring(0, functionStartIndex);
- }
- } catch (Function.FunctionFormatException ffe){
- // unknown function so
- // fallback to VALUE, and fullMetricName
- }
-
- List<Function> functionsList = new ArrayList<>();
- functionsList.add(function);
- metricsFunctions.put(cleanMetricName, functionsList);
- }
-
- return metricsFunctions;
- }
-
- @Override
- public TimelinePutResponse putMetrics(TimelineMetrics metrics) throws SQLException, IOException {
- // Error indicated by the Sql exception
- TimelinePutResponse response = new TimelinePutResponse();
-
- hBaseAccessor.insertMetricRecordsWithMetadata(metricMetadataManager, metrics, false);
-
- if (configuration.isCollectorInMemoryAggregationEnabled()) {
- cache.putMetrics(metrics.getMetrics(), metricMetadataManager);
- }
-
- return response;
- }
-
- @Override
- public TimelinePutResponse putContainerMetrics(List<ContainerMetric> metrics)
- throws SQLException, IOException {
-
- if (containerMetricsDisabled) {
- LOG.debug("Ignoring submitted container metrics according to configuration. Values will not be stored.");
- return new TimelinePutResponse();
- }
-
- hBaseAccessor.insertContainerMetrics(metrics);
- return new TimelinePutResponse();
- }
-
- @Override
- public Map<String, List<TimelineMetricMetadata>> getTimelineMetricMetadata(String appId, String metricPattern,
- boolean includeBlacklistedMetrics) throws SQLException, IOException {
- Map<TimelineMetricMetadataKey, TimelineMetricMetadata> metadata =
- metricMetadataManager.getMetadataCache();
-
- boolean filterByAppId = StringUtils.isNotEmpty(appId);
- boolean filterByMetricName = StringUtils.isNotEmpty(metricPattern);
- Pattern metricFilterPattern = null;
- if (filterByMetricName) {
- metricFilterPattern = Pattern.compile(metricPattern);
- }
-
- // Group Metadata by AppId
- Map<String, List<TimelineMetricMetadata>> metadataByAppId = new HashMap<>();
- for (TimelineMetricMetadata metricMetadata : metadata.values()) {
-
- if (!includeBlacklistedMetrics && !metricMetadata.isWhitelisted()) {
- continue;
- }
-
- String currentAppId = metricMetadata.getAppId();
- if (filterByAppId && !currentAppId.equals(appId)) {
- continue;
- }
-
- if (filterByMetricName) {
- Matcher m = metricFilterPattern.matcher(metricMetadata.getMetricName());
- if (!m.find()) {
- continue;
- }
- }
-
- List<TimelineMetricMetadata> metadataList = metadataByAppId.get(currentAppId);
- if (metadataList == null) {
- metadataList = new ArrayList<>();
- metadataByAppId.put(currentAppId, metadataList);
- }
-
- metadataList.add(metricMetadata);
- }
-
- return metadataByAppId;
- }
-
- @Override
- public byte[] getUuid(String metricName, String appId, String instanceId, String hostname) throws SQLException, IOException {
- return metricMetadataManager.getUuid(metricName, appId, instanceId, hostname);
- }
-
- @Override
- public Map<String, Set<String>> getHostAppsMetadata() throws SQLException, IOException {
- Map<String, TimelineMetricHostMetadata> hostsMetadata = metricMetadataManager.getHostedAppsCache();
- Map<String, Set<String>> hostAppMap = new HashMap<>();
- for (String hostname : hostsMetadata.keySet()) {
- hostAppMap.put(hostname, hostsMetadata.get(hostname).getHostedApps().keySet());
- }
- return hostAppMap;
- }
-
- @Override
- public TimelinePutResponse putHostAggregatedMetrics(AggregationResult aggregationResult) throws SQLException, IOException {
- Map<TimelineMetric, MetricHostAggregate> aggregateMap = new HashMap<>();
- String hostname = null;
- for (TimelineMetricWithAggregatedValues entry : aggregationResult.getResult()) {
- aggregateMap.put(entry.getTimelineMetric(), entry.getMetricAggregate());
- hostname = hostname == null ? entry.getTimelineMetric().getHostName() : hostname;
- break;
- }
- long timestamp = aggregationResult.getTimeInMilis();
- postedAggregatedMap.put(hostname, timestamp);
- if (LOG.isDebugEnabled()) {
- LOG.debug(String.format("Adding host %s to aggregated by in-memory aggregator. Timestamp : %s", hostname, timestamp));
- }
- hBaseAccessor.saveHostAggregateRecords(aggregateMap, PhoenixTransactSQL.METRICS_AGGREGATE_MINUTE_TABLE_NAME);
-
-
- return new TimelinePutResponse();
- }
-
- @Override
- public Map<String, Map<String,Set<String>>> getInstanceHostsMetadata(String instanceId, String appId) throws SQLException, IOException {
- Map<String, Set<String>> hostedApps = getHostAppsMetadata();
- Map<String, Set<String>> instanceHosts = metricMetadataManager.getHostedInstanceCache();
- if (configuration.getTimelineMetricsMultipleClusterSupport()) {
- instanceHosts = metricMetadataManager.getHostedInstanceCache();
- }
- Map<String, Map<String, Set<String>>> instanceAppHosts = new HashMap<>();
-
- if (MapUtils.isEmpty(instanceHosts)) {
- Map<String, Set<String>> appHostMap = new HashMap<String, Set<String>>();
- for (String host : hostedApps.keySet()) {
- for (String app : hostedApps.get(host)) {
- if (!appHostMap.containsKey(app)) {
- appHostMap.put(app, new HashSet<String>());
- }
- appHostMap.get(app).add(host);
- }
- }
- instanceAppHosts.put("", appHostMap);
- } else {
- for (String instance : instanceHosts.keySet()) {
-
- if (StringUtils.isNotEmpty(instanceId) && !instance.equals(instanceId)) {
- continue;
- }
- Map<String, Set<String>> appHostMap = new HashMap<String, Set<String>>();
- instanceAppHosts.put(instance, appHostMap);
-
- Set<String> hostsWithInstance = instanceHosts.get(instance);
- for (String host : hostsWithInstance) {
- for (String app : hostedApps.get(host)) {
- if (StringUtils.isNotEmpty(appId) && !app.equals(appId)) {
- continue;
- }
-
- if (!appHostMap.containsKey(app)) {
- appHostMap.put(app, new HashSet<String>());
- }
- appHostMap.get(app).add(host);
- }
- }
- }
- }
-
- return instanceAppHosts;
- }
-
- @Override
- public List<String> getLiveInstances() {
-
- List<String> instances = null;
- try {
- if (haController == null) {
- // Always return current host as live (embedded operation mode)
- return Collections.singletonList(configuration.getInstanceHostnameFromEnv());
- }
- instances = haController.getLiveInstanceHostNames();
- if (instances == null || instances.isEmpty()) {
- // fallback
- instances = Collections.singletonList(configuration.getInstanceHostnameFromEnv());
- }
- } catch (UnknownHostException e) {
- LOG.debug("Exception on getting hostname from env.", e);
- }
- return instances;
- }
-
- private void scheduleAggregatorThread(final TimelineMetricAggregator aggregator) {
- if (!aggregator.isDisabled()) {
- ScheduledExecutorService executorService = Executors.newSingleThreadScheduledExecutor(
- new ThreadFactory() {
- @Override
- public Thread newThread(Runnable r) {
- return new Thread(r, ACTUAL_AGGREGATOR_NAMES.get(aggregator.getName()));
- }
- }
- );
- scheduledExecutors.put(aggregator.getName(), executorService);
- executorService.scheduleAtFixedRate(aggregator,
- 0l,
- aggregator.getSleepIntervalMillis(),
- TimeUnit.MILLISECONDS);
- LOG.info("Scheduled aggregator thread " + aggregator.getName() + " every " +
- + aggregator.getSleepIntervalMillis() + " milliseconds.");
- } else {
- LOG.info("Skipped scheduling " + aggregator.getName() + " since it is disabled.");
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/MetricsCacheCommitterThread.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/MetricsCacheCommitterThread.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/MetricsCacheCommitterThread.java
deleted file mode 100644
index d858b84..0000000
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/MetricsCacheCommitterThread.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-
-public class MetricsCacheCommitterThread implements Runnable {
-
- private static final Log LOG = LogFactory.getLog(MetricsCacheCommitterThread.class);
- private static PhoenixHBaseAccessor phoenixHBaseAccessor;
-
- public MetricsCacheCommitterThread(PhoenixHBaseAccessor phoenixHBaseAccessor) {
- this.phoenixHBaseAccessor = phoenixHBaseAccessor;
- }
- @Override
- public void run() {
- LOG.debug("Checking if metrics cache is empty");
- if (!phoenixHBaseAccessor.isInsertCacheEmpty()) {
- phoenixHBaseAccessor.commitMetricsFromCache();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/MetricsSystemInitializationException.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/MetricsSystemInitializationException.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/MetricsSystemInitializationException.java
deleted file mode 100644
index 7bd4a32..0000000
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/MetricsSystemInitializationException.java
+++ /dev/null
@@ -1,42 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics
- .timeline;
-
-/**
- * RuntimeException for initialization of metrics schema. It is RuntimeException
- * since this is a not recoverable situation, and should be handled by main or
- * service method followed by shutdown.
- */
-public class MetricsSystemInitializationException extends RuntimeException {
- public MetricsSystemInitializationException() {
- }
-
- public MetricsSystemInitializationException(String msg) {
- super(msg);
- }
-
- public MetricsSystemInitializationException(Throwable t) {
- super(t);
- }
-
- public MetricsSystemInitializationException(String msg, Throwable t) {
- super(msg, t);
- }
-
-}
[25/28] ambari git commit: AMBARI-22740 : Fix integration test for
HBase in branch-3.0-ams due to UUID changes. (avijayan)
Posted by av...@apache.org.
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/TimelineMetricConfiguration.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/TimelineMetricConfiguration.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/TimelineMetricConfiguration.java
new file mode 100644
index 0000000..a1ad18c
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/TimelineMetricConfiguration.java
@@ -0,0 +1,712 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.metrics.core.timeline;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.net.InetAddress;
+import java.net.MalformedURLException;
+import java.net.URISyntaxException;
+import java.net.URL;
+import java.net.UnknownHostException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.ambari.metrics.core.timeline.source.InternalSourceProvider;
+import org.apache.commons.lang.StringUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.util.ReflectionUtils;
+import org.apache.ambari.metrics.core.timeline.sink.ExternalSinkProvider;
+import org.apache.ambari.metrics.core.timeline.source.DefaultInternalMetricsSourceProvider;
+import org.apache.log4j.Appender;
+import org.apache.log4j.FileAppender;
+import org.apache.log4j.Logger;
+
+/**
+ * Configuration class that reads properties from ams-site.xml. All values
+ * for time or intervals are given in seconds.
+ */
+public class TimelineMetricConfiguration {
+ private static final Log LOG = LogFactory.getLog(TimelineMetricConfiguration.class);
+
+ public static final String HBASE_SITE_CONFIGURATION_FILE = "hbase-site.xml";
+ public static final String METRICS_SITE_CONFIGURATION_FILE = "ams-site.xml";
+ public static final String METRICS_ENV_CONFIGURATION_FILE = "ams-env.xml";
+ public static final String METRICS_SSL_SERVER_CONFIGURATION_FILE = "ssl-server.xml";
+
+ public static final String TIMELINE_METRICS_AGGREGATOR_CHECKPOINT_DIR =
+ "timeline.metrics.aggregator.checkpoint.dir";
+
+ public static final String TIMELINE_METRIC_AGGREGATOR_SINK_CLASS =
+ "timeline.metrics.service.aggregator.sink.class";
+
+ public static final String TIMELINE_METRICS_SOURCE_PROVIDER_CLASS =
+ "timeline.metrics.service.source.provider.class";
+
+ public static final String TIMELINE_METRICS_SINK_PROVIDER_CLASS =
+ "timeline.metrics.service.sink.provider.class";
+
+ public static final String TIMELINE_METRICS_CACHE_SIZE =
+ "timeline.metrics.cache.size";
+
+ public static final String TIMELINE_METRICS_CACHE_COMMIT_INTERVAL =
+ "timeline.metrics.cache.commit.interval";
+
+ public static final String TIMELINE_METRICS_CACHE_ENABLED =
+ "timeline.metrics.cache.enabled";
+
+ public static final String DEFAULT_CHECKPOINT_LOCATION =
+ System.getProperty("java.io.tmpdir");
+
+ public static final String HBASE_ENCODING_SCHEME =
+ "timeline.metrics.hbase.data.block.encoding";
+
+ public static final String HBASE_COMPRESSION_SCHEME =
+ "timeline.metrics.hbase.compression.scheme";
+
+ public static final String CONTAINER_METRICS_TTL =
+ "timeline.container-metrics.ttl";
+
+ public static final String PRECISION_TABLE_TTL =
+ "timeline.metrics.host.aggregator.ttl";
+
+ public static final String HOST_MINUTE_TABLE_TTL =
+ "timeline.metrics.host.aggregator.minute.ttl";
+
+ public static final String HOST_DAILY_TABLE_TTL =
+ "timeline.metrics.host.aggregator.daily.ttl";
+
+ public static final String HOST_HOUR_TABLE_TTL =
+ "timeline.metrics.host.aggregator.hourly.ttl";
+
+ public static final String CLUSTER_SECOND_TABLE_TTL =
+ "timeline.metrics.cluster.aggregator.second.ttl";
+
+ public static final String CLUSTER_MINUTE_TABLE_TTL =
+ "timeline.metrics.cluster.aggregator.minute.ttl";
+
+ public static final String CLUSTER_HOUR_TABLE_TTL =
+ "timeline.metrics.cluster.aggregator.hourly.ttl";
+
+ public static final String CLUSTER_DAILY_TABLE_TTL =
+ "timeline.metrics.cluster.aggregator.daily.ttl";
+
+ public static final String CLUSTER_AGGREGATOR_TIMESLICE_INTERVAL =
+ "timeline.metrics.cluster.aggregator.second.timeslice.interval";
+
+ public static final String AGGREGATOR_CHECKPOINT_DELAY =
+ "timeline.metrics.service.checkpointDelay";
+
+ public static final String RESULTSET_FETCH_SIZE =
+ "timeline.metrics.service.resultset.fetchSize";
+
+ public static final String HOST_AGGREGATOR_MINUTE_SLEEP_INTERVAL =
+ "timeline.metrics.host.aggregator.minute.interval";
+
+ public static final String HOST_AGGREGATOR_HOUR_SLEEP_INTERVAL =
+ "timeline.metrics.host.aggregator.hourly.interval";
+
+ public static final String HOST_AGGREGATOR_DAILY_SLEEP_INTERVAL =
+ "timeline.metrics.host.aggregator.daily.interval";
+
+ public static final String CLUSTER_AGGREGATOR_SECOND_SLEEP_INTERVAL =
+ "timeline.metrics.cluster.aggregator.second.interval";
+
+ public static final String CLUSTER_AGGREGATOR_MINUTE_SLEEP_INTERVAL =
+ "timeline.metrics.cluster.aggregator.minute.interval";
+
+ public static final String CLUSTER_AGGREGATOR_HOUR_SLEEP_INTERVAL =
+ "timeline.metrics.cluster.aggregator.hourly.interval";
+
+ public static final String CLUSTER_AGGREGATOR_DAILY_SLEEP_INTERVAL =
+ "timeline.metrics.cluster.aggregator.daily.interval";
+
+ public static final String HOST_AGGREGATOR_MINUTE_CHECKPOINT_CUTOFF_MULTIPLIER =
+ "timeline.metrics.host.aggregator.minute.checkpointCutOffMultiplier";
+
+ public static final String HOST_AGGREGATOR_HOUR_CHECKPOINT_CUTOFF_MULTIPLIER =
+ "timeline.metrics.host.aggregator.hourly.checkpointCutOffMultiplier";
+
+ public static final String HOST_AGGREGATOR_DAILY_CHECKPOINT_CUTOFF_MULTIPLIER =
+ "timeline.metrics.host.aggregator.daily.checkpointCutOffMultiplier";
+
+ public static final String CLUSTER_AGGREGATOR_SECOND_CHECKPOINT_CUTOFF_MULTIPLIER =
+ "timeline.metrics.cluster.aggregator.second.checkpointCutOffMultiplier";
+
+ public static final String CLUSTER_AGGREGATOR_MINUTE_CHECKPOINT_CUTOFF_MULTIPLIER =
+ "timeline.metrics.cluster.aggregator.minute.checkpointCutOffMultiplier";
+
+ public static final String CLUSTER_AGGREGATOR_HOUR_CHECKPOINT_CUTOFF_MULTIPLIER =
+ "timeline.metrics.cluster.aggregator.hourly.checkpointCutOffMultiplier";
+
+ public static final String CLUSTER_AGGREGATOR_DAILY_CHECKPOINT_CUTOFF_MULTIPLIER =
+ "timeline.metrics.cluster.aggregator.daily.checkpointCutOffMultiplier";
+
+ public static final String GLOBAL_RESULT_LIMIT =
+ "timeline.metrics.service.default.result.limit";
+
+ public static final String GLOBAL_MAX_RETRIES =
+ "timeline.metrics.service.default.max_retries";
+
+ public static final String GLOBAL_RETRY_INTERVAL =
+ "timeline.metrics.service.default.retryInterval";
+
+ public static final String HOST_AGGREGATOR_MINUTE_DISABLED =
+ "timeline.metrics.host.aggregator.minute.disabled";
+
+ public static final String HOST_AGGREGATOR_HOUR_DISABLED =
+ "timeline.metrics.host.aggregator.hourly.disabled";
+
+ public static final String HOST_AGGREGATOR_DAILY_DISABLED =
+ "timeline.metrics.host.aggregator.hourly.disabled";
+
+ public static final String CLUSTER_AGGREGATOR_SECOND_DISABLED =
+ "timeline.metrics.cluster.aggregator.second.disabled";
+
+ public static final String CLUSTER_AGGREGATOR_MINUTE_DISABLED =
+ "timeline.metrics.cluster.aggregator.minute.disabled";
+
+ public static final String CLUSTER_AGGREGATOR_HOUR_DISABLED =
+ "timeline.metrics.cluster.aggregator.hourly.disabled";
+
+ public static final String CLUSTER_AGGREGATOR_DAILY_DISABLED =
+ "timeline.metrics.cluster.aggregator.daily.disabled";
+
+ public static final String WEBAPP_HTTP_ADDRESS =
+ "timeline.metrics.service.webapp.address";
+
+ public static final String TIMELINE_SERVICE_RPC_ADDRESS =
+ "timeline.metrics.service.rpc.address";
+
+ public static final String TIMELINE_SERVICE_DISABLE_CONTAINER_METRICS =
+ "timeline.metrics.service.container.metrics.disabled";
+
+ public static final String CLUSTER_AGGREGATOR_APP_IDS =
+ "timeline.metrics.service.cluster.aggregator.appIds";
+
+ public static final String SERVER_SIDE_TIMESIFT_ADJUSTMENT =
+ "timeline.metrics.service.cluster.aggregator.timeshift.adjustment";
+
+ public static final String OUT_OFF_BAND_DATA_TIME_ALLOWANCE =
+ "timeline.metrics.service.outofband.time.allowance.millis";
+
+ public static final String USE_GROUPBY_AGGREGATOR_QUERIES =
+ "timeline.metrics.service.use.groupBy.aggregators";
+
+ public static final String HANDLER_THREAD_COUNT =
+ "timeline.metrics.service.handler.thread.count";
+
+ public static final String WATCHER_DISABLED =
+ "timeline.metrics.service.watcher.disabled";
+
+ public static final String WATCHER_INITIAL_DELAY =
+ "timeline.metrics.service.watcher.initial.delay";
+
+ public static final String WATCHER_DELAY =
+ "timeline.metrics.service.watcher.delay";
+
+ public static final String WATCHER_TIMEOUT =
+ "timeline.metrics.service.watcher.timeout";
+
+ public static final String WATCHER_MAX_FAILURES =
+ "timeline.metrics.service.watcher.max.failures";
+
+ public static final String PRECISION_TABLE_SPLIT_POINTS =
+ "timeline.metrics.host.aggregate.splitpoints";
+
+ public static final String AGGREGATE_TABLE_SPLIT_POINTS =
+ "timeline.metrics.cluster.aggregate.splitpoints";
+
+ public static final String AGGREGATORS_SKIP_BLOCK_CACHE =
+ "timeline.metrics.aggregators.skip.blockcache.enabled";
+
+ public static final String TIMELINE_SERVICE_HTTP_POLICY =
+ "timeline.metrics.service.http.policy";
+
+ public static final String DISABLE_METRIC_METADATA_MGMT =
+ "timeline.metrics.service.metadata.management.disabled";
+
+ public static final String METRICS_METADATA_SYNC_INIT_DELAY =
+ "timeline.metrics.service.metadata.sync.init.delay";
+
+ public static final String METRICS_METADATA_SYNC_SCHEDULE_DELAY =
+ "timeline.metrics.service.metadata.sync.delay";
+
+ public static final String TIMELINE_METRICS_CLUSTER_AGGREGATOR_INTERPOLATION_ENABLED =
+ "timeline.metrics.cluster.aggregator.interpolation.enabled";
+
+ public static final String TIMELINE_METRICS_SINK_COLLECTION_PERIOD =
+ "timeline.metrics.sink.collection.period";
+
+ public static final String TIMELINE_METRICS_PRECISION_TABLE_DURABILITY =
+ "timeline.metrics.precision.table.durability";
+
+ public static final String TIMELINE_METRICS_AGGREGATE_TABLES_DURABILITY =
+ "timeline.metrics.aggregate.tables.durability";
+
+ public static final String TIMELINE_METRICS_WHITELIST_ENABLED =
+ "timeline.metrics.whitelisting.enabled";
+
+ public static final String TIMELINE_METRICS_WHITELIST_FILE =
+ "timeline.metrics.whitelist.file";
+
+ public static final String TIMELINE_METRICS_WHITELIST_FILE_LOCATION_DEFAULT =
+ "/etc/ambari-metrics-collector/conf/metrics_whitelist";
+
+ public static final String TIMELINE_METRIC_METADATA_FILTERS =
+ "timeline.metrics.service.metadata.filters";
+
+ public static final String TIMELINE_METRICS_APPS_BLACKLIST =
+ "timeline.metrics.apps.blacklist";
+
+ public static final String TIMELINE_METRICS_APPS_WHITELIST =
+ "timeline.metrics.apps.whitelist";
+
+ public static final String HBASE_BLOCKING_STORE_FILES =
+ "hbase.hstore.blockingStoreFiles";
+
+ public static final String DEFAULT_TOPN_HOSTS_LIMIT =
+ "timeline.metrics.default.topn.hosts.limit";
+
+ public static final String TIMELINE_METRIC_AGGREGATION_SQL_FILTERS =
+ "timeline.metrics.cluster.aggregation.sql.filters";
+
+ public static final String TIMELINE_METRICS_HBASE_AGGREGATE_TABLE_COMPACTION_POLICY_KEY =
+ "timeline.metrics.hbase.aggregate.table.compaction.policy.key";
+
+ public static final String TIMELINE_METRICS_HBASE_AGGREGATE_TABLE_COMPACTION_POLICY_CLASS =
+ "timeline.metrics.hbase.aggregate.table.compaction.policy.class";
+
+ public static final String TIMELINE_METRICS_AGGREGATE_TABLE_HBASE_BLOCKING_STORE_FILES =
+ "timeline.metrics.aggregate.table.hbase.hstore.blockingStoreFiles";
+
+ public static final String TIMELINE_METRICS_HBASE_PRECISION_TABLE_COMPACTION_POLICY_KEY =
+ "timeline.metrics.hbase.precision.table.compaction.policy.key";
+
+ public static final String TIMELINE_METRICS_HBASE_PRECISION_TABLE_COMPACTION_POLICY_CLASS =
+ "timeline.metrics.hbase.precision.table.compaction.policy.class";
+
+ public static final String TIMELINE_METRICS_PRECISION_TABLE_HBASE_BLOCKING_STORE_FILES =
+ "timeline.metrics.precision.table.hbase.hstore.blockingStoreFiles";
+
+ public static final String TIMELINE_METRICS_UUID_GEN_STRATEGY =
+ "timeline.metrics.uuid.gen.strategy";
+
+ public static final String TIMELINE_METRICS_SUPPORT_MULTIPLE_CLUSTERS =
+ "timeline.metrics.support.multiple.clusters";
+
+ public static final String HOST_APP_ID = "HOST";
+
+ public static final String DEFAULT_INSTANCE_PORT = "12001";
+
+ public static final String AMSHBASE_METRICS_WHITESLIST_FILE = "amshbase_metrics_whitelist";
+
+ public static final String TIMELINE_METRICS_HOST_INMEMORY_AGGREGATION = "timeline.metrics.host.inmemory.aggregation";
+
+ public static final String TIMELINE_METRICS_COLLECTOR_INMEMORY_AGGREGATION = "timeline.metrics.collector.inmemory.aggregation";
+
+ public static final String TIMELINE_METRICS_COLLECTOR_IGNITE_NODES = "timeline.metrics.collector.ignite.nodes.list";
+
+ public static final String TIMELINE_METRICS_COLLECTOR_IGNITE_BACKUPS = "timeline.metrics.collector.ignite.nodes.backups";
+
+ public static final String INTERNAL_CACHE_HEAP_PERCENT =
+ "timeline.metrics.internal.cache.%s.heap.percent";
+
+ public static final String EXTERNAL_SINK_INTERVAL =
+ "timeline.metrics.external.sink.%s.%s.interval";
+
+ public static final String DEFAULT_EXTERNAL_SINK_DIR =
+ "timeline.metrics.external.sink.dir";
+
+ public static final String KAFKA_SERVERS = "timeline.metrics.external.sink.kafka.bootstrap.servers";
+ public static final String KAFKA_ACKS = "timeline.metrics.external.sink.kafka.acks";
+ public static final String KAFKA_RETRIES = "timeline.metrics.external.sink.kafka.bootstrap.retries";
+ public static final String KAFKA_BATCH_SIZE = "timeline.metrics.external.sink.kafka.batch.size";
+ public static final String KAFKA_LINGER_MS = "timeline.metrics.external.sink.kafka.linger.ms";
+ public static final String KAFKA_BUFFER_MEM = "timeline.metrics.external.sink.kafka.buffer.memory";
+ public static final String KAFKA_SINK_TIMEOUT_SECONDS = "timeline.metrics.external.sink.kafka.timeout.seconds";
+
+ private Configuration hbaseConf;
+ private Configuration metricsConf;
+ private Configuration metricsSslConf;
+ private Configuration amsEnvConf;
+ private volatile boolean isInitialized = false;
+
+ private static TimelineMetricConfiguration instance = new TimelineMetricConfiguration();
+
+ private TimelineMetricConfiguration() {}
+
+ public static TimelineMetricConfiguration getInstance() {
+ return instance;
+ }
+
+ // Tests
+ public TimelineMetricConfiguration(Configuration hbaseConf, Configuration metricsConf) {
+ this.hbaseConf = hbaseConf;
+ this.metricsConf = metricsConf;
+ this.isInitialized = true;
+ }
+
+ public void initialize() throws URISyntaxException, MalformedURLException {
+ if (!isInitialized) {
+ ClassLoader classLoader = Thread.currentThread().getContextClassLoader();
+ if (classLoader == null) {
+ classLoader = getClass().getClassLoader();
+ }
+ URL hbaseResUrl = classLoader.getResource(HBASE_SITE_CONFIGURATION_FILE);
+ URL amsResUrl = classLoader.getResource(METRICS_SITE_CONFIGURATION_FILE);
+ LOG.info("Found hbase site configuration: " + hbaseResUrl);
+ LOG.info("Found metric service configuration: " + amsResUrl);
+
+ if (hbaseResUrl == null) {
+ throw new IllegalStateException("Unable to initialize the metrics " +
+ "subsystem. No hbase-site present in the classpath.");
+ }
+
+ if (amsResUrl == null) {
+ throw new IllegalStateException("Unable to initialize the metrics " +
+ "subsystem. No ams-site present in the classpath.");
+ }
+
+ hbaseConf = new Configuration(true);
+ hbaseConf.addResource(hbaseResUrl.toURI().toURL());
+ metricsConf = new Configuration(true);
+ metricsConf.addResource(amsResUrl.toURI().toURL());
+
+ if (metricsConf.get("timeline.metrics.service.http.policy", "HTTP_ONLY").equalsIgnoreCase("HTTPS_ONLY")) {
+ URL amsSllResUrl = classLoader.getResource(METRICS_SSL_SERVER_CONFIGURATION_FILE);
+ LOG.info("Found metric ssl service configuration: " + amsResUrl);
+ if (amsSllResUrl == null) {
+ throw new IllegalStateException("Unable to initialize the metrics " +
+ "subsystem. No ams-ssl-server present in the classpath.");
+ }
+ metricsSslConf = new Configuration(true);
+ metricsSslConf.addResource(amsSllResUrl.toURI().toURL());
+ }
+
+ isInitialized = true;
+ }
+ }
+
+ public Configuration getHbaseConf() throws URISyntaxException, MalformedURLException {
+ if (!isInitialized) {
+ initialize();
+ }
+ return hbaseConf;
+ }
+
+ public Configuration getMetricsConf() throws URISyntaxException, MalformedURLException {
+ if (!isInitialized) {
+ initialize();
+ }
+ return metricsConf;
+ }
+
+ public Configuration getMetricsSslConf() throws URISyntaxException, MalformedURLException {
+ if (!isInitialized) {
+ initialize();
+ }
+ return metricsSslConf;
+ }
+
+ public String getZKClientPort() throws MalformedURLException, URISyntaxException {
+ return getHbaseConf().getTrimmed("hbase.zookeeper.property.clientPort", "2181");
+ }
+
+ public String getZKQuorum() throws MalformedURLException, URISyntaxException {
+ return getHbaseConf().getTrimmed("hbase.zookeeper.quorum");
+ }
+
+ public String getClusterZKClientPort() throws MalformedURLException, URISyntaxException {
+ return getMetricsConf().getTrimmed("cluster.zookeeper.property.clientPort", "2181");
+ }
+
+ public String getClusterZKQuorum() throws MalformedURLException, URISyntaxException {
+ return getMetricsConf().getTrimmed("cluster.zookeeper.quorum");
+ }
+
+ public String getInstanceHostnameFromEnv() throws UnknownHostException {
+ String amsInstanceName = System.getProperty("AMS_INSTANCE_NAME");
+ if (amsInstanceName == null) {
+ amsInstanceName = InetAddress.getLocalHost().getHostName();
+ }
+ return amsInstanceName;
+ }
+
+ public String getInstancePort() throws MalformedURLException, URISyntaxException {
+ String amsInstancePort = System.getProperty("AMS_INSTANCE_PORT");
+ if (amsInstancePort == null) {
+ // Check config
+ return getMetricsConf().get("timeline.metrics.availability.instance.port", DEFAULT_INSTANCE_PORT);
+ }
+ return DEFAULT_INSTANCE_PORT;
+ }
+
+ public String getWebappAddress() throws MalformedURLException, URISyntaxException {
+ String defaultHttpAddress = "0.0.0.0:6188";
+ return getMetricsConf().get(WEBAPP_HTTP_ADDRESS, defaultHttpAddress);
+ }
+
+ public int getTimelineMetricsServiceHandlerThreadCount() {
+ if (metricsConf != null) {
+ return Integer.parseInt(metricsConf.get(HANDLER_THREAD_COUNT, "20"));
+ }
+ return 20;
+ }
+
+ public boolean isTimelineMetricsServiceWatcherDisabled() {
+ if (metricsConf != null) {
+ return Boolean.parseBoolean(metricsConf.get(WATCHER_DISABLED, "false"));
+ }
+ return false;
+ }
+
+ public int getTimelineMetricsServiceWatcherInitDelay() {
+ if (metricsConf != null) {
+ return Integer.parseInt(metricsConf.get(WATCHER_INITIAL_DELAY, "600"));
+ }
+ return 600;
+ }
+
+ public int getTimelineMetricsServiceWatcherDelay() {
+ if (metricsConf != null) {
+ return Integer.parseInt(metricsConf.get(WATCHER_DELAY, "30"));
+ }
+ return 30;
+ }
+
+ public int getTimelineMetricsServiceWatcherTimeout() {
+ if (metricsConf != null) {
+ return Integer.parseInt(metricsConf.get(WATCHER_TIMEOUT, "30"));
+ }
+ return 30;
+ }
+
+ public int getTimelineMetricsServiceWatcherMaxFailures() {
+ if (metricsConf != null) {
+ return Integer.parseInt(metricsConf.get(WATCHER_MAX_FAILURES, "3"));
+ }
+ return 3;
+ }
+
+ public boolean getTimelineMetricsMultipleClusterSupport() {
+ if (metricsConf != null) {
+ return Boolean.parseBoolean(metricsConf.get(TIMELINE_METRICS_SUPPORT_MULTIPLE_CLUSTERS, "false"));
+ }
+ return false;
+ }
+
+ public String getTimelineServiceRpcAddress() {
+ String defaultRpcAddress = "0.0.0.0:60200";
+ if (metricsConf != null) {
+ return metricsConf.get(TIMELINE_SERVICE_RPC_ADDRESS, defaultRpcAddress);
+ }
+ return defaultRpcAddress;
+ }
+
+ public String getKafkaServers() {
+ if (metricsConf != null) {
+ return metricsConf.get("timeline.metrics.kafka.servers", null);
+ }
+ return null;
+ }
+
+ public boolean isDistributedCollectorModeDisabled() {
+ try {
+ if (getMetricsConf() != null) {
+ return Boolean.parseBoolean(getMetricsConf().get("timeline.metrics.service.distributed.collector.mode.disabled", "false"));
+ }
+ return false;
+ } catch (Exception e) {
+ return false;
+ }
+ }
+
+ public boolean isSecurityEnabled() {
+ return hbaseConf.get("hbase.security.authentication", "").equals("kerberos");
+ }
+
+ public Set<String> getAmshbaseWhitelist() {
+
+ ClassLoader classLoader = Thread.currentThread().getContextClassLoader();
+ if (classLoader == null) {
+ classLoader = getClass().getClassLoader();
+ }
+
+ BufferedReader br = null;
+ String strLine;
+ Set<String> whitelist = new HashSet<>();
+
+ try(InputStream inputStream = classLoader.getResourceAsStream(AMSHBASE_METRICS_WHITESLIST_FILE)) {
+
+ if (inputStream == null) {
+ LOG.info("ams-hbase metrics whitelist file not present.");
+ return Collections.EMPTY_SET;
+ }
+
+ br = new BufferedReader(new InputStreamReader(inputStream));
+
+ while ((strLine = br.readLine()) != null) {
+ strLine = strLine.trim();
+ if (StringUtils.isEmpty(strLine)) {
+ continue;
+ }
+ whitelist.add(strLine);
+ }
+ } catch (Exception ex) {
+ LOG.error("Unable to read ams-hbase metric whitelist file", ex);
+ return Collections.EMPTY_SET;
+ }
+
+ return whitelist;
+ }
+
+ /**
+ * Get the sink interval for a metrics source.
+ * Determines how often the metrics will be written to the sink.
+ * This determines whether any caching will be needed on the collector
+ * side, default interval disables caching by writing at the same time as
+ * we get data.
+ *
+ * @param sinkProviderClassName Simple name of your implementation of {@link ExternalSinkProvider}
+ * @param sourceName {@link InternalSourceProvider.SOURCE_NAME}
+ * @return seconds
+ */
+ public int getExternalSinkInterval(String sinkProviderClassName,
+ InternalSourceProvider.SOURCE_NAME sourceName) {
+ String sinkProviderSimpleClassName = sinkProviderClassName.substring(
+ sinkProviderClassName.lastIndexOf(".") + 1);
+
+ return Integer.parseInt(metricsConf.get(
+ String.format(EXTERNAL_SINK_INTERVAL, sinkProviderSimpleClassName, sourceName), "-1"));
+ }
+
+ public InternalSourceProvider getInternalSourceProvider() {
+ Class<? extends InternalSourceProvider> providerClass =
+ metricsConf.getClass(TIMELINE_METRICS_SOURCE_PROVIDER_CLASS,
+ DefaultInternalMetricsSourceProvider.class, InternalSourceProvider.class);
+ return ReflectionUtils.newInstance(providerClass, metricsConf);
+ }
+
+ /**
+ * List of external sink provider classes. Comma-separated.
+ */
+ public List<ExternalSinkProvider> getExternalSinkProviderList() {
+ Class<?>[] providerClasses = metricsConf.getClasses(TIMELINE_METRICS_SINK_PROVIDER_CLASS);
+ List<ExternalSinkProvider> providerList = new ArrayList<>();
+ if (providerClasses != null) {
+ for (Class<?> providerClass : providerClasses) {
+ providerList.add((ExternalSinkProvider) ReflectionUtils.newInstance(providerClass, metricsConf));
+ }
+ }
+ return providerList;
+ }
+
+ public String getInternalCacheHeapPercent(String instanceName) {
+ String heapPercent = metricsConf.get(String.format(INTERNAL_CACHE_HEAP_PERCENT, instanceName));
+ if (StringUtils.isEmpty(heapPercent)) {
+ return "5%";
+ } else {
+ return heapPercent.endsWith("%") ? heapPercent : heapPercent + "%";
+ }
+ }
+
+ public String getDefaultMetricsSinkDir() {
+ String dirPath = metricsConf.get(DEFAULT_EXTERNAL_SINK_DIR);
+ if (dirPath == null) {
+ // Only one logger at the time of writing
+ Appender appender = (Appender) Logger.getRootLogger().getAllAppenders().nextElement();
+ if (appender instanceof FileAppender) {
+ File f = new File(((FileAppender) appender).getFile());
+ if (f.exists()) {
+ dirPath = f.getParent();
+ } else {
+ dirPath = "/tmp";
+ }
+ }
+ }
+
+ return dirPath;
+ }
+
+ public boolean isHostInMemoryAggregationEnabled() {
+ if (metricsConf != null) {
+ return Boolean.valueOf(metricsConf.get(TIMELINE_METRICS_HOST_INMEMORY_AGGREGATION, "false"));
+ } else {
+ return false;
+ }
+ }
+
+ public boolean isContainerMetricsDisabled() {
+ try {
+ return metricsConf != null && Boolean.parseBoolean(metricsConf.get(TIMELINE_SERVICE_DISABLE_CONTAINER_METRICS, "false"));
+ } catch (Exception e) {
+
+ return false;
+ }
+ }
+
+ public boolean isCollectorInMemoryAggregationEnabled() {
+ if (metricsConf != null) {
+ return Boolean.valueOf(metricsConf.get(TIMELINE_METRICS_COLLECTOR_INMEMORY_AGGREGATION, "false"));
+ } else {
+ return false;
+ }
+ }
+
+ public List<String> getAppIdsForHostAggregation() {
+ String appIds = metricsConf.get(CLUSTER_AGGREGATOR_APP_IDS);
+ if (!StringUtils.isEmpty(appIds)) {
+ return Arrays.asList(StringUtils.stripAll(appIds.split(",")));
+ }
+ return Collections.emptyList();
+ }
+
+ public String getZkConnectionUrl(String zkClientPort, String zkQuorum){
+ StringBuilder sb = new StringBuilder();
+ String[] quorumParts = zkQuorum.split(",");
+ String prefix = "";
+ for (String part : quorumParts) {
+ sb.append(prefix);
+ sb.append(part.trim());
+ if (!part.contains(":")) {
+ sb.append(":");
+ sb.append(zkClientPort);
+ }
+ prefix = ",";
+ }
+
+ return sb.toString();
+
+ }
+
+ public boolean isWhitelistingEnabled() {
+ if (metricsConf != null) {
+ return Boolean.parseBoolean(metricsConf.get(TIMELINE_METRICS_WHITELIST_ENABLED, "false"));
+ }
+ return false;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/TimelineMetricDistributedCache.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/TimelineMetricDistributedCache.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/TimelineMetricDistributedCache.java
new file mode 100644
index 0000000..000b3bc
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/TimelineMetricDistributedCache.java
@@ -0,0 +1,32 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.metrics.core.timeline;
+
+import java.util.Collection;
+import java.util.Map;
+
+import org.apache.hadoop.metrics2.sink.timeline.MetricClusterAggregate;
+import org.apache.hadoop.metrics2.sink.timeline.TimelineMetric;
+import org.apache.ambari.metrics.core.timeline.aggregators.TimelineClusterMetric;
+import org.apache.ambari.metrics.core.timeline.discovery.TimelineMetricMetadataManager;
+
+public interface TimelineMetricDistributedCache {
+ Map<TimelineClusterMetric, MetricClusterAggregate> evictMetricAggregates(Long startTime, Long endTime);
+ void putMetrics(Collection<TimelineMetric> elements, TimelineMetricMetadataManager metricMetadataManager);
+ Map<String, Double> getPointInTimeCacheMetrics();
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/TimelineMetricStore.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/TimelineMetricStore.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/TimelineMetricStore.java
new file mode 100644
index 0000000..901d51e
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/TimelineMetricStore.java
@@ -0,0 +1,110 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.metrics.core.timeline;
+
+import java.io.IOException;
+import java.sql.SQLException;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.hadoop.metrics2.sink.timeline.AggregationResult;
+import org.apache.hadoop.metrics2.sink.timeline.ContainerMetric;
+import org.apache.hadoop.metrics2.sink.timeline.Precision;
+import org.apache.hadoop.metrics2.sink.timeline.TimelineMetric;
+import org.apache.hadoop.metrics2.sink.timeline.TimelineMetricMetadata;
+import org.apache.hadoop.metrics2.sink.timeline.TimelineMetrics;
+import org.apache.hadoop.metrics2.sink.timeline.TopNConfig;
+import org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse;
+
+public interface TimelineMetricStore {
+ /**
+ * This method retrieves metrics stored by the Timeline store.
+ *
+ * @param metricNames Names of the metric, e.g.: cpu_user
+ * @param hostnames Names of the host where the metric originated from
+ * @param applicationId Id of the application to which this metric belongs
+ * @param instanceId Application instance id.
+ * @param startTime Start timestamp
+ * @param endTime End timestamp
+ * @param precision Precision [ seconds, minutes, hours ]
+ * @param limit Override default result limit
+ * @param groupedByHosts Group {@link TimelineMetric} by metric name, hostname,
+ * app id and instance id
+ * @param seriesAggregateFunction Specify this when caller want to aggregate multiple metrics
+ * series into one. [ SUM, AVG, MIN, MAX ]
+ *
+ * @return {@link TimelineMetric}
+ * @throws java.sql.SQLException
+ */
+ TimelineMetrics getTimelineMetrics(List<String> metricNames, List<String> hostnames,
+ String applicationId, String instanceId, Long startTime,
+ Long endTime, Precision precision, Integer limit, boolean groupedByHosts,
+ TopNConfig topNConfig, String seriesAggregateFunction)
+ throws SQLException, IOException;
+
+ /**
+ * Stores metric information to the timeline store. Any errors occurring for
+ * individual put request objects will be reported in the response.
+ *
+ * @param metrics An {@link TimelineMetrics}.
+ * @return An {@link org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse}.
+ * @throws SQLException, IOException
+ */
+ TimelinePutResponse putMetrics(TimelineMetrics metrics) throws SQLException, IOException;
+
+ /**
+ * Store container metric into the timeline tore
+ */
+ TimelinePutResponse putContainerMetrics(List<ContainerMetric> metrics)
+ throws SQLException, IOException;
+
+ /**
+ * Return all metrics metadata that have been written to the store.
+ * @return { appId : [ @TimelineMetricMetadata ] }
+ * @throws SQLException
+ * @throws IOException
+ */
+ Map<String, List<TimelineMetricMetadata>> getTimelineMetricMetadata(String appId, String metricPattern,
+ boolean includeBlacklistedMetrics) throws SQLException, IOException;
+
+ TimelinePutResponse putHostAggregatedMetrics(AggregationResult aggregationResult) throws SQLException, IOException;
+ /**
+ * Returns all hosts that have written metrics with the apps on the host
+ * @return { hostname : [ appIds ] }
+ * @throws SQLException
+ * @throws IOException
+ */
+ Map<String, Set<String>> getHostAppsMetadata() throws SQLException, IOException;
+
+ /**
+ * Returns all instances and the set of hosts each instance is present on
+ * @return { instanceId : [ hosts ] }
+ * @throws SQLException
+ * @throws IOException
+ */
+ Map<String, Map<String,Set<String>>> getInstanceHostsMetadata(String instanceId, String appId) throws SQLException, IOException;
+
+ byte[] getUuid(String metricName, String appId, String instanceId, String hostname) throws SQLException, IOException;
+
+ /**
+ * Return a list of known live collector nodes
+ * @return [ hostname ]
+ */
+ List<String> getLiveInstances();
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/TimelineMetricStoreWatcher.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/TimelineMetricStoreWatcher.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/TimelineMetricStoreWatcher.java
new file mode 100644
index 0000000..0ab7929
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/TimelineMetricStoreWatcher.java
@@ -0,0 +1,126 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.metrics.core.timeline;
+
+import java.util.Collections;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.metrics2.sink.timeline.Precision;
+import org.apache.hadoop.metrics2.sink.timeline.TimelineMetric;
+import org.apache.hadoop.metrics2.sink.timeline.TimelineMetrics;
+import org.apache.hadoop.util.ExitUtil;
+
+/**
+ * Acts as the single TimetineMetricStore Watcher.
+ */
+public class TimelineMetricStoreWatcher implements Runnable {
+
+ private static final Log LOG = LogFactory.getLog(TimelineMetricStoreWatcher.class);
+ private static final String FAKE_METRIC_NAME = "TimelineMetricStoreWatcher.FakeMetric";
+ private static final String FAKE_HOSTNAME = "fakehostname";
+ private static final String FAKE_APP_ID = "timeline_metric_store_watcher";
+
+ private static int failures = 0;
+ private final TimelineMetricConfiguration configuration;
+
+ private TimelineMetricStore timelineMetricStore;
+
+ //used to call timelineMetricStore blocking methods with timeout
+ private ExecutorService executor = Executors.newSingleThreadExecutor();
+
+
+ public TimelineMetricStoreWatcher(TimelineMetricStore timelineMetricStore,
+ TimelineMetricConfiguration configuration) {
+ this.timelineMetricStore = timelineMetricStore;
+ this.configuration = configuration;
+ }
+
+ @Override
+ public void run() {
+ if (checkMetricStore()) {
+ failures = 0;
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Successfully got metrics from TimelineMetricStore");
+ }
+ } else {
+ LOG.info("Failed to get metrics from TimelineMetricStore, attempt = " + failures);
+ failures++;
+ }
+
+ if (failures >= configuration.getTimelineMetricsServiceWatcherMaxFailures()) {
+ String msg = "Error getting metrics from TimelineMetricStore. " +
+ "Shutting down by TimelineMetricStoreWatcher.";
+ LOG.fatal(msg);
+ ExitUtil.terminate(-1, msg);
+ }
+
+ }
+
+ /**
+ * Checks TimelineMetricStore functionality by adding and getting
+ * a fake metric to/from HBase
+ * @return if check was successful
+ */
+ private boolean checkMetricStore() {
+ final long startTime = System.currentTimeMillis();
+ final int delay = configuration.getTimelineMetricsServiceWatcherDelay();
+ final int timeout = configuration.getTimelineMetricsServiceWatcherTimeout();
+
+ TimelineMetric fakeMetric = new TimelineMetric();
+ fakeMetric.setMetricName(FAKE_METRIC_NAME);
+ fakeMetric.setHostName(FAKE_HOSTNAME);
+ fakeMetric.setAppId(FAKE_APP_ID);
+ fakeMetric.setStartTime(startTime);
+ fakeMetric.getMetricValues().put(startTime, 0.0);
+
+ final TimelineMetrics metrics = new TimelineMetrics();
+ metrics.setMetrics(Collections.singletonList(fakeMetric));
+
+ Callable<TimelineMetric> task = new Callable<TimelineMetric>() {
+ public TimelineMetric call() throws Exception {
+ timelineMetricStore.putMetrics(metrics);
+ TimelineMetrics timelineMetrics = timelineMetricStore.getTimelineMetrics(
+ Collections.singletonList(FAKE_METRIC_NAME), Collections.singletonList(FAKE_HOSTNAME),
+ FAKE_APP_ID, null, startTime - delay * 2 * 1000,
+ startTime + delay * 2 * 1000, Precision.SECONDS, 1, true, null, null);
+ return timelineMetrics.getMetrics().get(0);
+ }
+ };
+
+ Future<TimelineMetric> future = executor.submit(task);
+ TimelineMetric timelineMetric = null;
+ try {
+ timelineMetric = future.get(timeout, TimeUnit.SECONDS);
+ // Phoenix might throw RuntimeExeption's
+ } catch (Exception e) {
+ return false;
+ } finally {
+ future.cancel(true);
+ }
+
+ return timelineMetric != null;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/TimelineMetricsAggregatorSink.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/TimelineMetricsAggregatorSink.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/TimelineMetricsAggregatorSink.java
new file mode 100644
index 0000000..bb379d8
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/TimelineMetricsAggregatorSink.java
@@ -0,0 +1,60 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.metrics.core.timeline;
+
+import java.util.Map;
+
+import org.apache.hadoop.metrics2.sink.timeline.MetricClusterAggregate;
+import org.apache.hadoop.metrics2.sink.timeline.MetricHostAggregate;
+import org.apache.hadoop.metrics2.sink.timeline.Precision;
+import org.apache.hadoop.metrics2.sink.timeline.TimelineMetric;
+import org.apache.ambari.metrics.core.timeline.aggregators.TimelineClusterMetric;
+
+/**
+ * This Interface for storing aggregated metrics to any external storage
+ */
+public interface TimelineMetricsAggregatorSink {
+
+ /**
+ * Save host aggregated metrics
+ *
+ * @param hostAggregateMap Map of host aggregated metrics
+ * @param precision SECOND, MINUTE, HOUR, DAY
+ */
+ void saveHostAggregateRecords(
+ Map<TimelineMetric, MetricHostAggregate> hostAggregateMap,
+ Precision precision);
+
+ /**
+ * Save cluster time aggregated metrics
+ *
+ * @param clusterTimeAggregateMap Map of cluster aggregated metrics
+ * @param precision SECOND, MINUTE, HOUR, DAY
+ */
+ void saveClusterTimeAggregateRecords(
+ Map<TimelineClusterMetric, MetricHostAggregate> clusterTimeAggregateMap,
+ Precision precision);
+
+ /**
+ * Save cluster aggregated metrics
+ *
+ * @param clusterAggregateMap Map of cluster aggregated metrics
+ */
+ void saveClusterAggregateRecords(
+ Map<TimelineClusterMetric, MetricClusterAggregate> clusterAggregateMap);
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/TimelineMetricsFilter.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/TimelineMetricsFilter.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/TimelineMetricsFilter.java
new file mode 100644
index 0000000..b2d5fd9
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/TimelineMetricsFilter.java
@@ -0,0 +1,165 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.metrics.core.timeline;
+
+import java.io.BufferedReader;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.Set;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.lang.StringUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.metrics2.sink.timeline.TimelineMetric;
+
+public class TimelineMetricsFilter {
+
+ private static Set<String> whitelistedMetrics;
+ private static Set<Pattern> whitelistedMetricPatterns;
+ private static Set<String> whitelistedApps;
+ private static Set<String> blacklistedApps;
+ private static String patternPrefix = "._p_";
+ private static Set<String> amshbaseWhitelist;
+
+ private static final Log LOG = LogFactory.getLog(TimelineMetricsFilter.class);
+
+ public static void initializeMetricFilter(TimelineMetricConfiguration configuration) {
+
+ Configuration metricsConf = null;
+ try {
+ metricsConf = configuration.getMetricsConf();
+ } catch (Exception e) {
+ LOG.error("Error fetching metrics configuration for getting whitelisting information");
+ return;
+ }
+
+ whitelistedMetrics = new HashSet<String>();
+ whitelistedMetricPatterns = new HashSet<Pattern>();
+ blacklistedApps = new HashSet<>();
+ whitelistedApps = new HashSet<>();
+ amshbaseWhitelist = new HashSet<>();
+
+ if (configuration.isWhitelistingEnabled()) {
+ String whitelistFile = metricsConf.get(TimelineMetricConfiguration.TIMELINE_METRICS_WHITELIST_FILE, TimelineMetricConfiguration.TIMELINE_METRICS_WHITELIST_FILE_LOCATION_DEFAULT);
+ readMetricWhitelistFromFile(whitelistFile);
+ }
+
+ String appsBlacklist = metricsConf.get(TimelineMetricConfiguration.TIMELINE_METRICS_APPS_BLACKLIST, "");
+ if (!StringUtils.isEmpty(appsBlacklist)) {
+ for (String app : appsBlacklist.split(",")) {
+ blacklistedApps.add(app);
+ }
+ LOG.info("Blacklisted apps : " + blacklistedApps.toString());
+ }
+
+ String appsWhitelist = metricsConf.get(TimelineMetricConfiguration.TIMELINE_METRICS_APPS_WHITELIST, "");
+ if (!StringUtils.isEmpty(appsWhitelist)) {
+ for (String app : appsWhitelist.split(",")) {
+ whitelistedApps.add(app);
+ }
+ LOG.info("Whitelisted apps : " + whitelistedApps.toString());
+ }
+
+ amshbaseWhitelist = configuration.getAmshbaseWhitelist();
+ if (CollectionUtils.isNotEmpty(amshbaseWhitelist)) {
+ LOG.info("Whitelisting " + amshbaseWhitelist.size() + " ams-hbase metrics");
+ }
+ }
+
+ private static void readMetricWhitelistFromFile(String whitelistFile) {
+
+ BufferedReader br = null;
+ String strLine;
+
+ try(FileInputStream fstream = new FileInputStream(whitelistFile)) {
+ br = new BufferedReader(new InputStreamReader(fstream));
+
+ while ((strLine = br.readLine()) != null) {
+ strLine = strLine.trim();
+ if (StringUtils.isEmpty(strLine)) {
+ continue;
+ }
+ if (strLine.startsWith(patternPrefix)) {
+ whitelistedMetricPatterns.add(Pattern.compile(strLine.substring(patternPrefix.length())));
+ } else {
+ whitelistedMetrics.add(strLine);
+ }
+ }
+ } catch (IOException ioEx) {
+ LOG.error("Unable to parse metric whitelist file", ioEx);
+ }
+
+ LOG.info("Whitelisting " + whitelistedMetrics.size() + " metrics");
+ LOG.debug("Whitelisted metrics : " + Arrays.toString(whitelistedMetrics.toArray()));
+ }
+
+ public static boolean acceptMetric(String metricName, String appId) {
+ TimelineMetric timelineMetric = new TimelineMetric();
+ timelineMetric.setAppId(appId);
+ timelineMetric.setMetricName(metricName);
+ return acceptMetric(timelineMetric);
+ }
+
+ public static boolean acceptMetric(TimelineMetric metric) {
+
+ String appId = metric.getAppId();
+ String metricName = metric.getMetricName();
+ // App Blacklisting
+ if (CollectionUtils.isNotEmpty(blacklistedApps) && blacklistedApps.contains(appId)) {
+ return false;
+ }
+
+ //Special Case appId = ams-hbase whitelisting.
+ if ("ams-hbase".equals(appId) && CollectionUtils.isNotEmpty(amshbaseWhitelist)) {
+ return amshbaseWhitelist.contains(metric.getMetricName());
+ }
+
+ // App Whitelisting
+ if (CollectionUtils.isNotEmpty(whitelistedApps) && whitelistedApps.contains(appId)) {
+ return true;
+ }
+
+ // Metric Whitelisting
+ if (CollectionUtils.isEmpty(whitelistedMetrics) && CollectionUtils.isEmpty(whitelistedMetricPatterns)) {
+ return true;
+ }
+
+ if (whitelistedMetrics.contains(metricName)) {
+ return true;
+ }
+
+ for (Pattern p : whitelistedMetricPatterns) {
+ Matcher m = p.matcher(metricName);
+ if (m.find()) {
+ whitelistedMetrics.add(metricName);
+ return true;
+ }
+ }
+
+ return false;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/TimelineMetricsIgniteCache.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/TimelineMetricsIgniteCache.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/TimelineMetricsIgniteCache.java
new file mode 100644
index 0000000..e6990e7
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/TimelineMetricsIgniteCache.java
@@ -0,0 +1,288 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.metrics.core.timeline;
+
+import static java.util.concurrent.TimeUnit.SECONDS;
+import static org.apache.ambari.metrics.core.timeline.aggregators.AggregatorUtils.getRoundedCheckPointTimeMillis;
+import static org.apache.ambari.metrics.core.timeline.aggregators.AggregatorUtils.getTimeSlices;
+import static org.apache.ambari.metrics.core.timeline.aggregators.AggregatorUtils.sliceFromTimelineMetric;
+
+import java.net.MalformedURLException;
+import java.net.URISyntaxException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.locks.Lock;
+
+import javax.cache.Cache;
+import javax.cache.expiry.CreatedExpiryPolicy;
+import javax.cache.expiry.Duration;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.metrics2.sink.timeline.MetricClusterAggregate;
+import org.apache.hadoop.metrics2.sink.timeline.TimelineMetric;
+import org.apache.hadoop.metrics2.sink.timeline.availability.MetricCollectorHAHelper;
+import org.apache.ambari.metrics.core.timeline.aggregators.TimelineClusterMetric;
+import org.apache.ambari.metrics.core.timeline.discovery.TimelineMetricHostMetadata;
+import org.apache.ambari.metrics.core.timeline.discovery.TimelineMetricMetadataManager;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.Ignition;
+import org.apache.ignite.cache.CacheAtomicityMode;
+import org.apache.ignite.cache.CacheMetrics;
+import org.apache.ignite.cache.CacheMode;
+import org.apache.ignite.cache.query.QueryCursor;
+import org.apache.ignite.cache.query.ScanQuery;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.lang.IgniteBiPredicate;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.ssl.SslContextFactory;
+
+public class TimelineMetricsIgniteCache implements TimelineMetricDistributedCache {
+ private static final Log LOG =
+ LogFactory.getLog(TimelineMetricsIgniteCache.class);
+ private IgniteCache<TimelineClusterMetric, MetricClusterAggregate> igniteCache;
+ private long cacheSliceIntervalMillis;
+ private boolean interpolationEnabled;
+ private List<String> skipAggrPatternStrings = new ArrayList<>();
+ private List<String> appIdsToAggregate;
+
+
+ public TimelineMetricsIgniteCache() throws MalformedURLException, URISyntaxException {
+ TimelineMetricConfiguration timelineMetricConfiguration = TimelineMetricConfiguration.getInstance();
+ Configuration metricConf = timelineMetricConfiguration.getMetricsConf();
+ Configuration sslConf = timelineMetricConfiguration.getMetricsSslConf();
+
+ IgniteConfiguration igniteConfiguration = new IgniteConfiguration();
+
+ //TODO add config to disable logging
+
+ //enable ssl for ignite requests
+ if (metricConf.get(TimelineMetricConfiguration.TIMELINE_SERVICE_HTTP_POLICY) != null && metricConf.get(TimelineMetricConfiguration.TIMELINE_SERVICE_HTTP_POLICY).equalsIgnoreCase("HTTPS_ONLY")) {
+ SslContextFactory sslContextFactory = new SslContextFactory();
+ String keyStorePath = sslConf.get("ssl.server.keystore.location");
+ String keyStorePassword = sslConf.get("ssl.server.keystore.password");
+ String trustStorePath = sslConf.get("ssl.server.truststore.location");
+ String trustStorePassword = sslConf.get("ssl.server.truststore.password");
+
+ sslContextFactory.setKeyStoreFilePath(keyStorePath);
+ sslContextFactory.setKeyStorePassword(keyStorePassword.toCharArray());
+ sslContextFactory.setTrustStoreFilePath(trustStorePath);
+ sslContextFactory.setTrustStorePassword(trustStorePassword.toCharArray());
+ igniteConfiguration.setSslContextFactory(sslContextFactory);
+ }
+
+ //aggregation parameters
+ appIdsToAggregate = timelineMetricConfiguration.getAppIdsForHostAggregation();
+ interpolationEnabled = Boolean.parseBoolean(metricConf.get(TimelineMetricConfiguration.TIMELINE_METRICS_CLUSTER_AGGREGATOR_INTERPOLATION_ENABLED, "true"));
+ cacheSliceIntervalMillis = SECONDS.toMillis(metricConf.getInt(TimelineMetricConfiguration.CLUSTER_AGGREGATOR_TIMESLICE_INTERVAL, 30));
+ Long aggregationInterval = metricConf.getLong(TimelineMetricConfiguration.CLUSTER_AGGREGATOR_SECOND_SLEEP_INTERVAL, 120L);
+
+ String filteredMetricPatterns = metricConf.get(TimelineMetricConfiguration.TIMELINE_METRIC_AGGREGATION_SQL_FILTERS);
+ if (!StringUtils.isEmpty(filteredMetricPatterns)) {
+ LOG.info("Skipping aggregation for metric patterns : " + filteredMetricPatterns);
+ skipAggrPatternStrings.addAll(Arrays.asList(filteredMetricPatterns.split(",")));
+ }
+
+ if (metricConf.get(TimelineMetricConfiguration.TIMELINE_METRICS_COLLECTOR_IGNITE_NODES) != null) {
+ TcpDiscoverySpi discoverySpi = new TcpDiscoverySpi();
+ TcpDiscoveryVmIpFinder ipFinder = new TcpDiscoveryVmIpFinder();
+ ipFinder.setAddresses(Arrays.asList(metricConf.get(TimelineMetricConfiguration.TIMELINE_METRICS_COLLECTOR_IGNITE_NODES).split(",")));
+ LOG.info("Setting ignite nodes to : " + ipFinder.getRegisteredAddresses());
+ discoverySpi.setIpFinder(ipFinder);
+ igniteConfiguration.setDiscoverySpi(discoverySpi);
+ } else {
+ //get live nodes from ZK
+ String zkClientPort = timelineMetricConfiguration.getClusterZKClientPort();
+ String zkQuorum = timelineMetricConfiguration.getClusterZKQuorum();
+ String zkConnectionURL = timelineMetricConfiguration.getZkConnectionUrl(zkClientPort, zkQuorum);
+ MetricCollectorHAHelper metricCollectorHAHelper = new MetricCollectorHAHelper(zkConnectionURL, 5, 200);
+ Collection<String> liveCollectors = metricCollectorHAHelper.findLiveCollectorHostsFromZNode();
+ if (liveCollectors != null && !liveCollectors.isEmpty()) {
+ TcpDiscoverySpi discoverySpi = new TcpDiscoverySpi();
+ TcpDiscoveryVmIpFinder ipFinder = new TcpDiscoveryVmIpFinder();
+ ipFinder.setAddresses(liveCollectors);
+ LOG.info("Setting ignite nodes to : " + ipFinder.getRegisteredAddresses());
+ discoverySpi.setIpFinder(ipFinder);
+ igniteConfiguration.setDiscoverySpi(discoverySpi);
+ }
+ }
+
+
+ //ignite cache configuration
+ CacheConfiguration<TimelineClusterMetric, MetricClusterAggregate> cacheConfiguration = new CacheConfiguration<>();
+ cacheConfiguration.setName("metrics_cache");
+ //set cache mode to partitioned with # of backups
+ cacheConfiguration.setCacheMode(CacheMode.PARTITIONED);
+ cacheConfiguration.setBackups(metricConf.getInt(TimelineMetricConfiguration.TIMELINE_METRICS_COLLECTOR_IGNITE_BACKUPS, 1));
+ //disable throttling due to cpu impact
+ cacheConfiguration.setRebalanceThrottle(0);
+ //enable locks
+ cacheConfiguration.setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL);
+ //expiry policy to remove lost keys, if any
+ cacheConfiguration.setEagerTtl(true);
+ cacheConfiguration.setExpiryPolicyFactory(CreatedExpiryPolicy.factoryOf(new Duration(TimeUnit.SECONDS, aggregationInterval * 3)));
+
+ Ignite igniteNode = Ignition.start(igniteConfiguration);
+ igniteCache = igniteNode.getOrCreateCache(cacheConfiguration);
+ }
+
+ /**
+ * Looks through the cache and evicts all elements within (startTime; endTime] half-interval
+ * All elements satisfying the half-interval will be removed from the cache.
+ * @param startTime
+ * @param endTime
+ * @return
+ */
+ @Override
+ public Map<TimelineClusterMetric, MetricClusterAggregate> evictMetricAggregates(Long startTime, Long endTime) {
+ Map<TimelineClusterMetric, MetricClusterAggregate> aggregatedMetricsMap = new HashMap<>();
+
+ //construct filter
+ IgniteBiPredicate<TimelineClusterMetric, MetricClusterAggregate> filter =
+ (IgniteBiPredicate<TimelineClusterMetric, MetricClusterAggregate>) (key, value) -> key.getTimestamp() > startTime && key.getTimestamp() <= endTime;
+
+ //get values from cache
+ try (QueryCursor<Cache.Entry<TimelineClusterMetric, MetricClusterAggregate>> cursor = igniteCache.query(new ScanQuery(filter))) {
+ for (Cache.Entry<TimelineClusterMetric, MetricClusterAggregate> e : cursor) {
+ aggregatedMetricsMap.put(e.getKey(), e.getValue());
+ }
+ }
+
+ //remove values from cache
+ igniteCache.removeAllAsync(aggregatedMetricsMap.keySet());
+
+ return aggregatedMetricsMap;
+ }
+
+ /**
+ * Iterates through elements skipping white-listed patterns;
+ * calculates average value for each slice of each metric (last slice values could be ignored in there is the possibility that values from this slice could be present in next post);
+ * updates/adds the value in the cache;
+ * calculates applications host metrics based on the metadata of hosted apps
+ * updates metadata of hosted apps if needed
+ * @param elements
+ * @param metadataManager
+ */
+ @Override
+ public void putMetrics(Collection<TimelineMetric> elements, TimelineMetricMetadataManager metadataManager) {
+ Map<String, TimelineMetricHostMetadata> hostMetadata = metadataManager.getHostedAppsCache();
+ for (TimelineMetric metric : elements) {
+ if (shouldBeSkipped(metric.getMetricName())) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug(String.format("Skipping %s metric from being aggregated", metric.getMetricName()));
+ }
+ continue;
+ }
+ List<Long[]> timeSlices = getTimeSlices(getRoundedCheckPointTimeMillis(metric.getMetricValues().firstKey(), cacheSliceIntervalMillis), metric.getMetricValues().lastKey(), cacheSliceIntervalMillis);
+ Map<TimelineClusterMetric, Double> slicedClusterMetrics = sliceFromTimelineMetric(metric, timeSlices, interpolationEnabled);
+
+ if (slicedClusterMetrics != null) {
+ for (Map.Entry<TimelineClusterMetric, Double> metricDoubleEntry : slicedClusterMetrics.entrySet()) {
+ MetricClusterAggregate newMetricClusterAggregate = new MetricClusterAggregate(
+ metricDoubleEntry.getValue(), 1, null, metricDoubleEntry.getValue(), metricDoubleEntry.getValue());
+ //put app metric into cache
+ putMetricIntoCache(metricDoubleEntry.getKey(), newMetricClusterAggregate);
+ if (hostMetadata != null) {
+ //calculate app host metric
+ if (metric.getAppId().equalsIgnoreCase(TimelineMetricConfiguration.HOST_APP_ID)) {
+ // Candidate metric, update app aggregates
+ if (hostMetadata.containsKey(metric.getHostName())) {
+ updateAppAggregatesFromHostMetric(metricDoubleEntry.getKey(), newMetricClusterAggregate, hostMetadata.get(metric.getHostName()));
+ }
+ } else {
+ // Build the hostedapps map if not a host metric
+ // Check app candidacy for host aggregation
+ //TODO better to lock TimelineMetricHostMetadata instance to avoid dataloss, but generally the data could be lost only during initial collector start
+ if (appIdsToAggregate.contains(metric.getAppId())) {
+ TimelineMetricHostMetadata timelineMetricHostMetadata = hostMetadata.get(metric.getHostName());
+ ConcurrentHashMap<String, String> appIdsMap;
+ if (timelineMetricHostMetadata == null) {
+ appIdsMap = new ConcurrentHashMap<>();
+ hostMetadata.put(metric.getHostName(), new TimelineMetricHostMetadata(appIdsMap));
+ } else {
+ appIdsMap = timelineMetricHostMetadata.getHostedApps();
+ }
+ if (!appIdsMap.containsKey(metric.getAppId())) {
+ appIdsMap.put(metric.getAppId(), metric.getAppId());
+ LOG.info("Adding appId to hosted apps: appId = " +
+ metric.getAppId() + ", hostname = " + metric.getHostName());
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+
+ private void updateAppAggregatesFromHostMetric(TimelineClusterMetric key, MetricClusterAggregate newMetricClusterAggregate, TimelineMetricHostMetadata timelineMetricHostMetadata) {
+ for (String appId : timelineMetricHostMetadata.getHostedApps().keySet()) {
+ TimelineClusterMetric timelineClusterMetric = new TimelineClusterMetric(key.getMetricName(), appId, key.getInstanceId(), key.getTimestamp());
+ putMetricIntoCache(timelineClusterMetric, newMetricClusterAggregate);
+ }
+ }
+
+ private void putMetricIntoCache(TimelineClusterMetric metricKey, MetricClusterAggregate metricValue) {
+ Lock lock = igniteCache.lock(metricKey);
+ lock.lock();
+ try {
+ MetricClusterAggregate metricClusterAggregateFromCache = igniteCache.get(metricKey);
+ if (metricClusterAggregateFromCache == null) {
+ igniteCache.put(metricKey, metricValue);
+ } else {
+ metricClusterAggregateFromCache.updateAggregates(metricValue);
+ igniteCache.put(metricKey, metricClusterAggregateFromCache);
+ }
+ } catch (Exception e) {
+ LOG.error("Exception : ", e);
+ } finally {
+ lock.unlock();
+ }
+ }
+
+ @Override
+ public Map<String, Double> getPointInTimeCacheMetrics() {
+ CacheMetrics clusterIgniteMetrics = igniteCache.metrics();
+ Map<String, Double> metricsMap = new HashMap<>();
+ metricsMap.put("Cluster_AverageGetTime", (double) clusterIgniteMetrics.getAverageGetTime());
+ metricsMap.put("Cluster_AveragePutTime", (double) clusterIgniteMetrics.getAveragePutTime());
+ metricsMap.put("Cluster_KeySize", (double) clusterIgniteMetrics.getKeySize());
+ metricsMap.put("Cluster_OffHeapAllocatedSize", (double) clusterIgniteMetrics.getOffHeapAllocatedSize());
+ return metricsMap;
+ }
+
+ private boolean shouldBeSkipped(String metricName) {
+ for (String pattern : skipAggrPatternStrings) {
+ if (metricName.matches(pattern)) {
+ return true;
+ }
+ }
+ return false;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/aggregators/AbstractTimelineAggregator.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/aggregators/AbstractTimelineAggregator.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/aggregators/AbstractTimelineAggregator.java
new file mode 100644
index 0000000..f538bdf
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/aggregators/AbstractTimelineAggregator.java
@@ -0,0 +1,494 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.metrics.core.timeline.aggregators;
+
+import static java.util.concurrent.TimeUnit.SECONDS;
+import static org.apache.ambari.metrics.core.timeline.aggregators.AggregatorUtils.getRoundedAggregateTimeMillis;
+import static org.apache.ambari.metrics.core.timeline.aggregators.AggregatorUtils.getRoundedCheckPointTimeMillis;
+import static org.apache.ambari.metrics.core.timeline.availability.AggregationTaskRunner.ACTUAL_AGGREGATOR_NAMES;
+
+import java.io.File;
+import java.io.IOException;
+import java.sql.Connection;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.util.Date;
+import java.util.Iterator;
+import java.util.List;
+
+import org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguration;
+import org.apache.ambari.metrics.core.timeline.availability.MetricCollectorHAController;
+import org.apache.ambari.metrics.core.timeline.query.Condition;
+import org.apache.ambari.metrics.core.timeline.query.EmptyCondition;
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.ambari.metrics.core.timeline.PhoenixHBaseAccessor;
+import org.apache.ambari.metrics.core.timeline.availability.AggregationTaskRunner;
+import org.apache.ambari.metrics.core.timeline.availability.AggregationTaskRunner.AGGREGATOR_NAME;
+import org.apache.ambari.metrics.core.timeline.query.PhoenixTransactSQL;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Base class for all runnable aggregators. Provides common functions like
+ * check pointing and scheduling.
+ */
+public abstract class AbstractTimelineAggregator implements TimelineMetricAggregator {
+ protected final PhoenixHBaseAccessor hBaseAccessor;
+ protected final Logger LOG;
+ protected final long checkpointDelayMillis;
+ protected final Integer resultsetFetchSize;
+ protected Configuration metricsConf;
+ private String checkpointLocation;
+ private Long sleepIntervalMillis;
+ private Integer checkpointCutOffMultiplier;
+ private String aggregatorDisableParam;
+ protected String tableName;
+ protected String outputTableName;
+ protected Long nativeTimeRangeDelay;
+ protected AggregationTaskRunner taskRunner;
+ protected List<String> downsampleMetricPatterns;
+ protected List<CustomDownSampler> configuredDownSamplers;
+
+ // Explicitly name aggregators for logging needs
+ private final AGGREGATOR_NAME aggregatorName;
+
+ AbstractTimelineAggregator(AGGREGATOR_NAME aggregatorName,
+ PhoenixHBaseAccessor hBaseAccessor,
+ Configuration metricsConf) {
+ this.aggregatorName = aggregatorName;
+ this.hBaseAccessor = hBaseAccessor;
+ this.metricsConf = metricsConf;
+ this.checkpointDelayMillis = SECONDS.toMillis(metricsConf.getInt(TimelineMetricConfiguration.AGGREGATOR_CHECKPOINT_DELAY, 120));
+ this.resultsetFetchSize = metricsConf.getInt(TimelineMetricConfiguration.RESULTSET_FETCH_SIZE, 2000);
+ this.LOG = LoggerFactory.getLogger(ACTUAL_AGGREGATOR_NAMES.get(aggregatorName));
+ this.configuredDownSamplers = DownSamplerUtils.getDownSamplers(metricsConf);
+ this.downsampleMetricPatterns = DownSamplerUtils.getDownsampleMetricPatterns(metricsConf);
+ }
+
+ public AbstractTimelineAggregator(AGGREGATOR_NAME aggregatorName,
+ PhoenixHBaseAccessor hBaseAccessor,
+ Configuration metricsConf,
+ String checkpointLocation,
+ Long sleepIntervalMillis,
+ Integer checkpointCutOffMultiplier,
+ String aggregatorDisableParam,
+ String tableName,
+ String outputTableName,
+ Long nativeTimeRangeDelay,
+ MetricCollectorHAController haController) {
+ this(aggregatorName, hBaseAccessor, metricsConf);
+ this.checkpointLocation = checkpointLocation;
+ this.sleepIntervalMillis = sleepIntervalMillis;
+ this.checkpointCutOffMultiplier = checkpointCutOffMultiplier;
+ this.aggregatorDisableParam = aggregatorDisableParam;
+ this.tableName = tableName;
+ this.outputTableName = outputTableName;
+ this.nativeTimeRangeDelay = nativeTimeRangeDelay;
+ this.taskRunner = haController != null && haController.isInitialized() ?
+ haController.getAggregationTaskRunner() : null;
+ }
+
+ @Override
+ public void run() {
+ LOG.info("Started Timeline aggregator thread @ " + new Date());
+ Long SLEEP_INTERVAL = getSleepIntervalMillis();
+ runOnce(SLEEP_INTERVAL);
+ }
+
+ /**
+ * Access relaxed for tests
+ */
+ public void runOnce(Long SLEEP_INTERVAL) {
+ boolean performAggregationFunction = true;
+ if (taskRunner != null) {
+ switch (getAggregatorType()) {
+ case HOST:
+ performAggregationFunction = taskRunner.performsHostAggregation();
+ break;
+ case CLUSTER:
+ performAggregationFunction = taskRunner.performsClusterAggregation();
+ }
+ }
+
+ if (performAggregationFunction) {
+ long currentTime = System.currentTimeMillis();
+ long lastCheckPointTime = readLastCheckpointSavingOnFirstRun(currentTime);
+
+ if (lastCheckPointTime != -1) {
+ LOG.info("Last check point time: " + lastCheckPointTime + ", lagBy: "
+ + ((currentTime - lastCheckPointTime) / 1000)
+ + " seconds.");
+
+ boolean success = doWork(lastCheckPointTime, lastCheckPointTime + SLEEP_INTERVAL);
+
+ if (success) {
+ try {
+ saveCheckPoint(lastCheckPointTime + SLEEP_INTERVAL);
+ } catch (IOException io) {
+ LOG.warn("Error saving checkpoint, restarting aggregation at " +
+ "previous checkpoint.");
+ }
+ }
+ }
+ } else {
+ LOG.info("Skipping aggregation function not owned by this instance.");
+ }
+ }
+
+ private long readLastCheckpointSavingOnFirstRun(long currentTime) {
+ long lastCheckPointTime = -1;
+
+ try {
+ lastCheckPointTime = readCheckPoint();
+ if (lastCheckPointTime != -1) {
+ LOG.info("Last Checkpoint read : " + new Date(lastCheckPointTime));
+ if (isLastCheckPointTooOld(currentTime, lastCheckPointTime)) {
+ LOG.warn("Last Checkpoint is too old, discarding last checkpoint. " +
+ "lastCheckPointTime = " + new Date(lastCheckPointTime));
+ lastCheckPointTime = getRoundedAggregateTimeMillis(getSleepIntervalMillis()) - getSleepIntervalMillis();
+ LOG.info("Saving checkpoint time. " + new Date((lastCheckPointTime)));
+ saveCheckPoint(lastCheckPointTime);
+
+ } else {
+
+ if (lastCheckPointTime > 0) {
+ lastCheckPointTime = getRoundedCheckPointTimeMillis(lastCheckPointTime, getSleepIntervalMillis());
+ LOG.info("Rounded off checkpoint : " + new Date(lastCheckPointTime));
+ }
+
+ if (isLastCheckPointTooYoung(lastCheckPointTime)) {
+ LOG.info("Last checkpoint too recent for aggregation. Sleeping for 1 cycle.");
+ return -1; //Skip Aggregation this time around
+ }
+ }
+ } else {
+ /*
+ No checkpoint. Save current rounded checkpoint and sleep for 1 cycle.
+ */
+ LOG.info("No checkpoint found");
+ long firstCheckPoint = getRoundedAggregateTimeMillis(getSleepIntervalMillis());
+ LOG.info("Saving checkpoint time. " + new Date((firstCheckPoint)));
+ saveCheckPoint(firstCheckPoint);
+ }
+ } catch (IOException io) {
+ LOG.warn("Unable to write last checkpoint time. Resuming sleep.", io);
+ }
+ return lastCheckPointTime;
+ }
+
+ private boolean isLastCheckPointTooOld(long currentTime, long checkpoint) {
+ // first checkpoint is saved checkpointDelayMillis in the past,
+ // so here we also need to take it into account
+ return checkpoint != -1 &&
+ ((currentTime - checkpoint) > getCheckpointCutOffIntervalMillis());
+ }
+
+ private boolean isLastCheckPointTooYoung(long checkpoint) {
+ return checkpoint != -1 &&
+ ((getRoundedAggregateTimeMillis(getSleepIntervalMillis()) <= checkpoint));
+ }
+
+ protected long readCheckPoint() {
+ if (taskRunner != null) {
+ return taskRunner.getCheckpointManager().readCheckpoint(aggregatorName);
+ }
+ try {
+ File checkpoint = new File(getCheckpointLocation());
+ if (checkpoint.exists()) {
+ String contents = FileUtils.readFileToString(checkpoint);
+ if (contents != null && !contents.isEmpty()) {
+ return Long.parseLong(contents);
+ }
+ }
+ } catch (IOException io) {
+ LOG.debug("", io);
+ }
+ return -1;
+ }
+
+ protected void saveCheckPoint(long checkpointTime) throws IOException {
+ if (taskRunner != null) {
+ boolean success = taskRunner.getCheckpointManager().writeCheckpoint(aggregatorName, checkpointTime);
+ if (!success) {
+ LOG.error("Error saving checkpoint with AggregationTaskRunner, " +
+ "aggregator = " + aggregatorName + "value = " + checkpointTime);
+ }
+ } else {
+ File checkpoint = new File(getCheckpointLocation());
+ if (!checkpoint.exists()) {
+ boolean done = checkpoint.createNewFile();
+ if (!done) {
+ throw new IOException("Could not create checkpoint at location, " +
+ getCheckpointLocation());
+ }
+ }
+ FileUtils.writeStringToFile(checkpoint, String.valueOf(checkpointTime));
+ }
+ }
+
+ /**
+ * Read metrics written during the time interval and save the sum and total
+ * in the aggregate table.
+ *
+ * @param startTime Sample start time
+ * @param endTime Sample end time
+ */
+ public boolean doWork(long startTime, long endTime) {
+ LOG.info("Start aggregation cycle @ " + new Date() + ", " +
+ "startTime = " + new Date(startTime) + ", endTime = " + new Date(endTime));
+
+ boolean success = true;
+ Condition condition = prepareMetricQueryCondition(startTime, endTime);
+
+ Connection conn = null;
+ PreparedStatement stmt = null;
+ ResultSet rs = null;
+
+ try {
+ conn = hBaseAccessor.getConnection();
+ // FLUME 2. aggregate and ignore the instance
+ stmt = PhoenixTransactSQL.prepareGetMetricsSqlStmt(conn, condition);
+
+ LOG.debug("Query issued @: " + new Date());
+ if (condition.doUpdate()) {
+ int rows = stmt.executeUpdate();
+ conn.commit();
+ LOG.info(rows + " row(s) updated in aggregation.");
+
+ //TODO : Fix downsampling after UUID change.
+ //downsample(conn, startTime, endTime);
+ } else {
+ rs = stmt.executeQuery();
+ }
+ LOG.debug("Query returned @: " + new Date());
+
+ aggregate(rs, startTime, endTime);
+
+ } catch (Exception e) {
+ LOG.error("Exception during aggregating metrics.", e);
+ success = false;
+ } finally {
+ if (rs != null) {
+ try {
+ rs.close();
+ } catch (SQLException e) {
+ // Ignore
+ }
+ }
+ if (stmt != null) {
+ try {
+ stmt.close();
+ } catch (SQLException e) {
+ // Ignore
+ }
+ }
+ if (conn != null) {
+ try {
+ conn.close();
+ } catch (SQLException sql) {
+ // Ignore
+ }
+ }
+ }
+
+ LOG.info("End aggregation cycle @ " + new Date());
+ return success;
+ }
+
+ protected abstract Condition prepareMetricQueryCondition(long startTime, long endTime);
+
+ protected abstract void aggregate(ResultSet rs, long startTime, long endTime) throws IOException, SQLException;
+
+ protected void downsample(Connection conn, Long startTime, Long endTime) {
+
+ LOG.debug("Checking for downsampling requests.");
+ if (CollectionUtils.isEmpty(configuredDownSamplers)) {
+ LOG.debug("No downsamplers configured");
+ return;
+ }
+
+ // Generate UPSERT query prefix. UPSERT part of the query is needed on the Aggregator side.
+ // SELECT part of the query is provided by the downsampler.
+ String queryPrefix = PhoenixTransactSQL.DOWNSAMPLE_CLUSTER_METRIC_SQL_UPSERT_PREFIX;
+ if (outputTableName.contains("RECORD")) {
+ queryPrefix = PhoenixTransactSQL.DOWNSAMPLE_HOST_METRIC_SQL_UPSERT_PREFIX;
+ }
+ queryPrefix = String.format(queryPrefix, outputTableName);
+
+ for (Iterator<CustomDownSampler> iterator = configuredDownSamplers.iterator(); iterator.hasNext();){
+ CustomDownSampler downSampler = iterator.next();
+
+ if (downSampler.validateConfigs()) {
+ EmptyCondition downSamplingCondition = new EmptyCondition();
+ downSamplingCondition.setDoUpdate(true);
+ List<String> stmts = downSampler.prepareDownSamplingStatement(startTime, endTime, tableName);
+ for (String stmt : stmts) {
+ downSamplingCondition.setStatement(queryPrefix + stmt);
+ runDownSamplerQuery(conn, downSamplingCondition);
+ }
+ } else {
+ LOG.warn("The following downsampler failed config validation : " + downSampler.getClass().getName() + "." +
+ "Removing it from downsamplers list.");
+ iterator.remove();
+ }
+ }
+
+ }
+
+ public Long getSleepIntervalMillis() {
+ return sleepIntervalMillis;
+ }
+
+ public void setSleepIntervalMillis(Long sleepIntervalMillis) {
+ this.sleepIntervalMillis = sleepIntervalMillis;
+ }
+
+ protected Integer getCheckpointCutOffMultiplier() {
+ return checkpointCutOffMultiplier;
+ }
+
+ protected Long getCheckpointCutOffIntervalMillis() {
+ return getCheckpointCutOffMultiplier() * getSleepIntervalMillis();
+ }
+
+ public boolean isDisabled() {
+ return metricsConf.getBoolean(aggregatorDisableParam, false);
+ }
+
+ protected String getQueryHint(Long startTime) {
+ StringBuilder sb = new StringBuilder();
+ sb.append("/*+ ");
+ sb.append("NATIVE_TIME_RANGE(");
+ sb.append(startTime - nativeTimeRangeDelay);
+ sb.append(") ");
+ if (hBaseAccessor.isSkipBlockCacheForAggregatorsEnabled()) {
+ sb.append("NO_CACHE ");
+ }
+ sb.append("*/");
+ return sb.toString();
+ }
+
+ protected String getCheckpointLocation() {
+ return checkpointLocation;
+ }
+
+ /**
+ * Run 1 downsampler query.
+ * @param conn
+ * @param condition
+ */
+ private void runDownSamplerQuery(Connection conn, Condition condition) {
+
+ PreparedStatement stmt = null;
+ ResultSet rs = null;
+ LOG.debug("Downsampling query : " + condition.getStatement());
+
+ try {
+ stmt = PhoenixTransactSQL.prepareGetMetricsSqlStmt(conn, condition);
+
+ LOG.debug("Downsampler Query issued...");
+ if (condition.doUpdate()) {
+ int rows = stmt.executeUpdate();
+ conn.commit();
+ LOG.info(rows + " row(s) updated in downsampling.");
+ } else {
+ rs = stmt.executeQuery();
+ }
+ LOG.debug("Downsampler Query returned ...");
+ LOG.info("End Downsampling cycle.");
+
+ } catch (SQLException e) {
+ LOG.error("Exception during downsampling metrics.", e);
+ } finally {
+ if (rs != null) {
+ try {
+ rs.close();
+ } catch (SQLException e) {
+ // Ignore
+ }
+ }
+ if (stmt != null) {
+ try {
+ stmt.close();
+ } catch (SQLException e) {
+ // Ignore
+ }
+ }
+ if (conn != null) {
+ try {
+ conn.close();
+ } catch (SQLException sql) {
+ // Ignore
+ }
+ }
+ }
+ }
+
+ /**
+ * Returns the METRIC_NAME NOT LIKE clause if certain metrics or metric patterns are to be skipped
+ * since they will be downsampled.
+ * @return
+ */
+ protected String getDownsampledMetricSkipClause() {
+
+ //TODO Fix downsampling for UUID change.
+ return StringUtils.EMPTY;
+
+// if (CollectionUtils.isEmpty(this.downsampleMetricPatterns)) {
+// return StringUtils.EMPTY;
+// }
+//
+// StringBuilder sb = new StringBuilder();
+//
+// for (int i = 0; i < downsampleMetricPatterns.size(); i++) {
+// sb.append(" METRIC_NAME");
+// sb.append(" NOT");
+// sb.append(" LIKE ");
+// sb.append("'" + downsampleMetricPatterns.get(i) + "'");
+//
+// if (i < downsampleMetricPatterns.size() - 1) {
+// sb.append(" AND ");
+// }
+// }
+//
+// sb.append(" AND ");
+// return sb.toString();
+ }
+
+ /**
+ * Get @AGGREGATOR_TYPE based on the output table.
+ * This is solely used by the HAController to determine which lock to acquire.
+ */
+ public AGGREGATOR_TYPE getAggregatorType() {
+ if (outputTableName.contains("RECORD")) {
+ return AGGREGATOR_TYPE.HOST;
+ } else if (outputTableName.contains("AGGREGATE")) {
+ return AGGREGATOR_TYPE.CLUSTER;
+ }
+ return null;
+ }
+
+ @Override
+ public AGGREGATOR_NAME getName() {
+ return aggregatorName;
+ }
+}
[16/28] ambari git commit: AMBARI-22740 : Fix integration test for
HBase in branch-3.0-ams due to UUID changes. (avijayan)
Posted by av...@apache.org.
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricConfiguration.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricConfiguration.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricConfiguration.java
deleted file mode 100644
index d4d8bb1..0000000
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricConfiguration.java
+++ /dev/null
@@ -1,713 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline;
-
-import java.io.BufferedReader;
-import java.io.File;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.net.InetAddress;
-import java.net.MalformedURLException;
-import java.net.URISyntaxException;
-import java.net.URL;
-import java.net.UnknownHostException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-
-import org.apache.commons.lang.StringUtils;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.util.ReflectionUtils;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.sink.ExternalSinkProvider;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.source.DefaultInternalMetricsSourceProvider;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.source.InternalSourceProvider;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.source.InternalSourceProvider.SOURCE_NAME;
-import org.apache.log4j.Appender;
-import org.apache.log4j.FileAppender;
-import org.apache.log4j.Logger;
-
-/**
- * Configuration class that reads properties from ams-site.xml. All values
- * for time or intervals are given in seconds.
- */
-public class TimelineMetricConfiguration {
- private static final Log LOG = LogFactory.getLog(TimelineMetricConfiguration.class);
-
- public static final String HBASE_SITE_CONFIGURATION_FILE = "hbase-site.xml";
- public static final String METRICS_SITE_CONFIGURATION_FILE = "ams-site.xml";
- public static final String METRICS_ENV_CONFIGURATION_FILE = "ams-env.xml";
- public static final String METRICS_SSL_SERVER_CONFIGURATION_FILE = "ssl-server.xml";
-
- public static final String TIMELINE_METRICS_AGGREGATOR_CHECKPOINT_DIR =
- "timeline.metrics.aggregator.checkpoint.dir";
-
- public static final String TIMELINE_METRIC_AGGREGATOR_SINK_CLASS =
- "timeline.metrics.service.aggregator.sink.class";
-
- public static final String TIMELINE_METRICS_SOURCE_PROVIDER_CLASS =
- "timeline.metrics.service.source.provider.class";
-
- public static final String TIMELINE_METRICS_SINK_PROVIDER_CLASS =
- "timeline.metrics.service.sink.provider.class";
-
- public static final String TIMELINE_METRICS_CACHE_SIZE =
- "timeline.metrics.cache.size";
-
- public static final String TIMELINE_METRICS_CACHE_COMMIT_INTERVAL =
- "timeline.metrics.cache.commit.interval";
-
- public static final String TIMELINE_METRICS_CACHE_ENABLED =
- "timeline.metrics.cache.enabled";
-
- public static final String DEFAULT_CHECKPOINT_LOCATION =
- System.getProperty("java.io.tmpdir");
-
- public static final String HBASE_ENCODING_SCHEME =
- "timeline.metrics.hbase.data.block.encoding";
-
- public static final String HBASE_COMPRESSION_SCHEME =
- "timeline.metrics.hbase.compression.scheme";
-
- public static final String CONTAINER_METRICS_TTL =
- "timeline.container-metrics.ttl";
-
- public static final String PRECISION_TABLE_TTL =
- "timeline.metrics.host.aggregator.ttl";
-
- public static final String HOST_MINUTE_TABLE_TTL =
- "timeline.metrics.host.aggregator.minute.ttl";
-
- public static final String HOST_DAILY_TABLE_TTL =
- "timeline.metrics.host.aggregator.daily.ttl";
-
- public static final String HOST_HOUR_TABLE_TTL =
- "timeline.metrics.host.aggregator.hourly.ttl";
-
- public static final String CLUSTER_SECOND_TABLE_TTL =
- "timeline.metrics.cluster.aggregator.second.ttl";
-
- public static final String CLUSTER_MINUTE_TABLE_TTL =
- "timeline.metrics.cluster.aggregator.minute.ttl";
-
- public static final String CLUSTER_HOUR_TABLE_TTL =
- "timeline.metrics.cluster.aggregator.hourly.ttl";
-
- public static final String CLUSTER_DAILY_TABLE_TTL =
- "timeline.metrics.cluster.aggregator.daily.ttl";
-
- public static final String CLUSTER_AGGREGATOR_TIMESLICE_INTERVAL =
- "timeline.metrics.cluster.aggregator.second.timeslice.interval";
-
- public static final String AGGREGATOR_CHECKPOINT_DELAY =
- "timeline.metrics.service.checkpointDelay";
-
- public static final String RESULTSET_FETCH_SIZE =
- "timeline.metrics.service.resultset.fetchSize";
-
- public static final String HOST_AGGREGATOR_MINUTE_SLEEP_INTERVAL =
- "timeline.metrics.host.aggregator.minute.interval";
-
- public static final String HOST_AGGREGATOR_HOUR_SLEEP_INTERVAL =
- "timeline.metrics.host.aggregator.hourly.interval";
-
- public static final String HOST_AGGREGATOR_DAILY_SLEEP_INTERVAL =
- "timeline.metrics.host.aggregator.daily.interval";
-
- public static final String CLUSTER_AGGREGATOR_SECOND_SLEEP_INTERVAL =
- "timeline.metrics.cluster.aggregator.second.interval";
-
- public static final String CLUSTER_AGGREGATOR_MINUTE_SLEEP_INTERVAL =
- "timeline.metrics.cluster.aggregator.minute.interval";
-
- public static final String CLUSTER_AGGREGATOR_HOUR_SLEEP_INTERVAL =
- "timeline.metrics.cluster.aggregator.hourly.interval";
-
- public static final String CLUSTER_AGGREGATOR_DAILY_SLEEP_INTERVAL =
- "timeline.metrics.cluster.aggregator.daily.interval";
-
- public static final String HOST_AGGREGATOR_MINUTE_CHECKPOINT_CUTOFF_MULTIPLIER =
- "timeline.metrics.host.aggregator.minute.checkpointCutOffMultiplier";
-
- public static final String HOST_AGGREGATOR_HOUR_CHECKPOINT_CUTOFF_MULTIPLIER =
- "timeline.metrics.host.aggregator.hourly.checkpointCutOffMultiplier";
-
- public static final String HOST_AGGREGATOR_DAILY_CHECKPOINT_CUTOFF_MULTIPLIER =
- "timeline.metrics.host.aggregator.daily.checkpointCutOffMultiplier";
-
- public static final String CLUSTER_AGGREGATOR_SECOND_CHECKPOINT_CUTOFF_MULTIPLIER =
- "timeline.metrics.cluster.aggregator.second.checkpointCutOffMultiplier";
-
- public static final String CLUSTER_AGGREGATOR_MINUTE_CHECKPOINT_CUTOFF_MULTIPLIER =
- "timeline.metrics.cluster.aggregator.minute.checkpointCutOffMultiplier";
-
- public static final String CLUSTER_AGGREGATOR_HOUR_CHECKPOINT_CUTOFF_MULTIPLIER =
- "timeline.metrics.cluster.aggregator.hourly.checkpointCutOffMultiplier";
-
- public static final String CLUSTER_AGGREGATOR_DAILY_CHECKPOINT_CUTOFF_MULTIPLIER =
- "timeline.metrics.cluster.aggregator.daily.checkpointCutOffMultiplier";
-
- public static final String GLOBAL_RESULT_LIMIT =
- "timeline.metrics.service.default.result.limit";
-
- public static final String GLOBAL_MAX_RETRIES =
- "timeline.metrics.service.default.max_retries";
-
- public static final String GLOBAL_RETRY_INTERVAL =
- "timeline.metrics.service.default.retryInterval";
-
- public static final String HOST_AGGREGATOR_MINUTE_DISABLED =
- "timeline.metrics.host.aggregator.minute.disabled";
-
- public static final String HOST_AGGREGATOR_HOUR_DISABLED =
- "timeline.metrics.host.aggregator.hourly.disabled";
-
- public static final String HOST_AGGREGATOR_DAILY_DISABLED =
- "timeline.metrics.host.aggregator.hourly.disabled";
-
- public static final String CLUSTER_AGGREGATOR_SECOND_DISABLED =
- "timeline.metrics.cluster.aggregator.second.disabled";
-
- public static final String CLUSTER_AGGREGATOR_MINUTE_DISABLED =
- "timeline.metrics.cluster.aggregator.minute.disabled";
-
- public static final String CLUSTER_AGGREGATOR_HOUR_DISABLED =
- "timeline.metrics.cluster.aggregator.hourly.disabled";
-
- public static final String CLUSTER_AGGREGATOR_DAILY_DISABLED =
- "timeline.metrics.cluster.aggregator.daily.disabled";
-
- public static final String WEBAPP_HTTP_ADDRESS =
- "timeline.metrics.service.webapp.address";
-
- public static final String TIMELINE_SERVICE_RPC_ADDRESS =
- "timeline.metrics.service.rpc.address";
-
- public static final String TIMELINE_SERVICE_DISABLE_CONTAINER_METRICS =
- "timeline.metrics.service.container.metrics.disabled";
-
- public static final String CLUSTER_AGGREGATOR_APP_IDS =
- "timeline.metrics.service.cluster.aggregator.appIds";
-
- public static final String SERVER_SIDE_TIMESIFT_ADJUSTMENT =
- "timeline.metrics.service.cluster.aggregator.timeshift.adjustment";
-
- public static final String OUT_OFF_BAND_DATA_TIME_ALLOWANCE =
- "timeline.metrics.service.outofband.time.allowance.millis";
-
- public static final String USE_GROUPBY_AGGREGATOR_QUERIES =
- "timeline.metrics.service.use.groupBy.aggregators";
-
- public static final String HANDLER_THREAD_COUNT =
- "timeline.metrics.service.handler.thread.count";
-
- public static final String WATCHER_DISABLED =
- "timeline.metrics.service.watcher.disabled";
-
- public static final String WATCHER_INITIAL_DELAY =
- "timeline.metrics.service.watcher.initial.delay";
-
- public static final String WATCHER_DELAY =
- "timeline.metrics.service.watcher.delay";
-
- public static final String WATCHER_TIMEOUT =
- "timeline.metrics.service.watcher.timeout";
-
- public static final String WATCHER_MAX_FAILURES =
- "timeline.metrics.service.watcher.max.failures";
-
- public static final String PRECISION_TABLE_SPLIT_POINTS =
- "timeline.metrics.host.aggregate.splitpoints";
-
- public static final String AGGREGATE_TABLE_SPLIT_POINTS =
- "timeline.metrics.cluster.aggregate.splitpoints";
-
- public static final String AGGREGATORS_SKIP_BLOCK_CACHE =
- "timeline.metrics.aggregators.skip.blockcache.enabled";
-
- public static final String TIMELINE_SERVICE_HTTP_POLICY =
- "timeline.metrics.service.http.policy";
-
- public static final String DISABLE_METRIC_METADATA_MGMT =
- "timeline.metrics.service.metadata.management.disabled";
-
- public static final String METRICS_METADATA_SYNC_INIT_DELAY =
- "timeline.metrics.service.metadata.sync.init.delay";
-
- public static final String METRICS_METADATA_SYNC_SCHEDULE_DELAY =
- "timeline.metrics.service.metadata.sync.delay";
-
- public static final String TIMELINE_METRICS_CLUSTER_AGGREGATOR_INTERPOLATION_ENABLED =
- "timeline.metrics.cluster.aggregator.interpolation.enabled";
-
- public static final String TIMELINE_METRICS_SINK_COLLECTION_PERIOD =
- "timeline.metrics.sink.collection.period";
-
- public static final String TIMELINE_METRICS_PRECISION_TABLE_DURABILITY =
- "timeline.metrics.precision.table.durability";
-
- public static final String TIMELINE_METRICS_AGGREGATE_TABLES_DURABILITY =
- "timeline.metrics.aggregate.tables.durability";
-
- public static final String TIMELINE_METRICS_WHITELIST_ENABLED =
- "timeline.metrics.whitelisting.enabled";
-
- public static final String TIMELINE_METRICS_WHITELIST_FILE =
- "timeline.metrics.whitelist.file";
-
- public static final String TIMELINE_METRICS_WHITELIST_FILE_LOCATION_DEFAULT =
- "/etc/ambari-metrics-collector/conf/metrics_whitelist";
-
- public static final String TIMELINE_METRIC_METADATA_FILTERS =
- "timeline.metrics.service.metadata.filters";
-
- public static final String TIMELINE_METRICS_APPS_BLACKLIST =
- "timeline.metrics.apps.blacklist";
-
- public static final String TIMELINE_METRICS_APPS_WHITELIST =
- "timeline.metrics.apps.whitelist";
-
- public static final String HBASE_BLOCKING_STORE_FILES =
- "hbase.hstore.blockingStoreFiles";
-
- public static final String DEFAULT_TOPN_HOSTS_LIMIT =
- "timeline.metrics.default.topn.hosts.limit";
-
- public static final String TIMELINE_METRIC_AGGREGATION_SQL_FILTERS =
- "timeline.metrics.cluster.aggregation.sql.filters";
-
- public static final String TIMELINE_METRICS_HBASE_AGGREGATE_TABLE_COMPACTION_POLICY_KEY =
- "timeline.metrics.hbase.aggregate.table.compaction.policy.key";
-
- public static final String TIMELINE_METRICS_HBASE_AGGREGATE_TABLE_COMPACTION_POLICY_CLASS =
- "timeline.metrics.hbase.aggregate.table.compaction.policy.class";
-
- public static final String TIMELINE_METRICS_AGGREGATE_TABLE_HBASE_BLOCKING_STORE_FILES =
- "timeline.metrics.aggregate.table.hbase.hstore.blockingStoreFiles";
-
- public static final String TIMELINE_METRICS_HBASE_PRECISION_TABLE_COMPACTION_POLICY_KEY =
- "timeline.metrics.hbase.precision.table.compaction.policy.key";
-
- public static final String TIMELINE_METRICS_HBASE_PRECISION_TABLE_COMPACTION_POLICY_CLASS =
- "timeline.metrics.hbase.precision.table.compaction.policy.class";
-
- public static final String TIMELINE_METRICS_PRECISION_TABLE_HBASE_BLOCKING_STORE_FILES =
- "timeline.metrics.precision.table.hbase.hstore.blockingStoreFiles";
-
- public static final String TIMELINE_METRICS_UUID_GEN_STRATEGY =
- "timeline.metrics.uuid.gen.strategy";
-
- public static final String TIMELINE_METRICS_SUPPORT_MULTIPLE_CLUSTERS =
- "timeline.metrics.support.multiple.clusters";
-
- public static final String HOST_APP_ID = "HOST";
-
- public static final String DEFAULT_INSTANCE_PORT = "12001";
-
- public static final String AMSHBASE_METRICS_WHITESLIST_FILE = "amshbase_metrics_whitelist";
-
- public static final String TIMELINE_METRICS_HOST_INMEMORY_AGGREGATION = "timeline.metrics.host.inmemory.aggregation";
-
- public static final String TIMELINE_METRICS_COLLECTOR_INMEMORY_AGGREGATION = "timeline.metrics.collector.inmemory.aggregation";
-
- public static final String TIMELINE_METRICS_COLLECTOR_IGNITE_NODES = "timeline.metrics.collector.ignite.nodes.list";
-
- public static final String TIMELINE_METRICS_COLLECTOR_IGNITE_BACKUPS = "timeline.metrics.collector.ignite.nodes.backups";
-
- public static final String INTERNAL_CACHE_HEAP_PERCENT =
- "timeline.metrics.internal.cache.%s.heap.percent";
-
- public static final String EXTERNAL_SINK_INTERVAL =
- "timeline.metrics.external.sink.%s.%s.interval";
-
- public static final String DEFAULT_EXTERNAL_SINK_DIR =
- "timeline.metrics.external.sink.dir";
-
- public static final String KAFKA_SERVERS = "timeline.metrics.external.sink.kafka.bootstrap.servers";
- public static final String KAFKA_ACKS = "timeline.metrics.external.sink.kafka.acks";
- public static final String KAFKA_RETRIES = "timeline.metrics.external.sink.kafka.bootstrap.retries";
- public static final String KAFKA_BATCH_SIZE = "timeline.metrics.external.sink.kafka.batch.size";
- public static final String KAFKA_LINGER_MS = "timeline.metrics.external.sink.kafka.linger.ms";
- public static final String KAFKA_BUFFER_MEM = "timeline.metrics.external.sink.kafka.buffer.memory";
- public static final String KAFKA_SINK_TIMEOUT_SECONDS = "timeline.metrics.external.sink.kafka.timeout.seconds";
-
- private Configuration hbaseConf;
- private Configuration metricsConf;
- private Configuration metricsSslConf;
- private Configuration amsEnvConf;
- private volatile boolean isInitialized = false;
-
- private static TimelineMetricConfiguration instance = new TimelineMetricConfiguration();
-
- private TimelineMetricConfiguration() {}
-
- public static TimelineMetricConfiguration getInstance() {
- return instance;
- }
-
- // Tests
- public TimelineMetricConfiguration(Configuration hbaseConf, Configuration metricsConf) {
- this.hbaseConf = hbaseConf;
- this.metricsConf = metricsConf;
- this.isInitialized = true;
- }
-
- public void initialize() throws URISyntaxException, MalformedURLException {
- if (!isInitialized) {
- ClassLoader classLoader = Thread.currentThread().getContextClassLoader();
- if (classLoader == null) {
- classLoader = getClass().getClassLoader();
- }
- URL hbaseResUrl = classLoader.getResource(HBASE_SITE_CONFIGURATION_FILE);
- URL amsResUrl = classLoader.getResource(METRICS_SITE_CONFIGURATION_FILE);
- LOG.info("Found hbase site configuration: " + hbaseResUrl);
- LOG.info("Found metric service configuration: " + amsResUrl);
-
- if (hbaseResUrl == null) {
- throw new IllegalStateException("Unable to initialize the metrics " +
- "subsystem. No hbase-site present in the classpath.");
- }
-
- if (amsResUrl == null) {
- throw new IllegalStateException("Unable to initialize the metrics " +
- "subsystem. No ams-site present in the classpath.");
- }
-
- hbaseConf = new Configuration(true);
- hbaseConf.addResource(hbaseResUrl.toURI().toURL());
- metricsConf = new Configuration(true);
- metricsConf.addResource(amsResUrl.toURI().toURL());
-
- if (metricsConf.get("timeline.metrics.service.http.policy", "HTTP_ONLY").equalsIgnoreCase("HTTPS_ONLY")) {
- URL amsSllResUrl = classLoader.getResource(METRICS_SSL_SERVER_CONFIGURATION_FILE);
- LOG.info("Found metric ssl service configuration: " + amsResUrl);
- if (amsSllResUrl == null) {
- throw new IllegalStateException("Unable to initialize the metrics " +
- "subsystem. No ams-ssl-server present in the classpath.");
- }
- metricsSslConf = new Configuration(true);
- metricsSslConf.addResource(amsSllResUrl.toURI().toURL());
- }
-
- isInitialized = true;
- }
- }
-
- public Configuration getHbaseConf() throws URISyntaxException, MalformedURLException {
- if (!isInitialized) {
- initialize();
- }
- return hbaseConf;
- }
-
- public Configuration getMetricsConf() throws URISyntaxException, MalformedURLException {
- if (!isInitialized) {
- initialize();
- }
- return metricsConf;
- }
-
- public Configuration getMetricsSslConf() throws URISyntaxException, MalformedURLException {
- if (!isInitialized) {
- initialize();
- }
- return metricsSslConf;
- }
-
- public String getZKClientPort() throws MalformedURLException, URISyntaxException {
- return getHbaseConf().getTrimmed("hbase.zookeeper.property.clientPort", "2181");
- }
-
- public String getZKQuorum() throws MalformedURLException, URISyntaxException {
- return getHbaseConf().getTrimmed("hbase.zookeeper.quorum");
- }
-
- public String getClusterZKClientPort() throws MalformedURLException, URISyntaxException {
- return getMetricsConf().getTrimmed("cluster.zookeeper.property.clientPort", "2181");
- }
-
- public String getClusterZKQuorum() throws MalformedURLException, URISyntaxException {
- return getMetricsConf().getTrimmed("cluster.zookeeper.quorum");
- }
-
- public String getInstanceHostnameFromEnv() throws UnknownHostException {
- String amsInstanceName = System.getProperty("AMS_INSTANCE_NAME");
- if (amsInstanceName == null) {
- amsInstanceName = InetAddress.getLocalHost().getHostName();
- }
- return amsInstanceName;
- }
-
- public String getInstancePort() throws MalformedURLException, URISyntaxException {
- String amsInstancePort = System.getProperty("AMS_INSTANCE_PORT");
- if (amsInstancePort == null) {
- // Check config
- return getMetricsConf().get("timeline.metrics.availability.instance.port", DEFAULT_INSTANCE_PORT);
- }
- return DEFAULT_INSTANCE_PORT;
- }
-
- public String getWebappAddress() throws MalformedURLException, URISyntaxException {
- String defaultHttpAddress = "0.0.0.0:6188";
- return getMetricsConf().get(WEBAPP_HTTP_ADDRESS, defaultHttpAddress);
- }
-
- public int getTimelineMetricsServiceHandlerThreadCount() {
- if (metricsConf != null) {
- return Integer.parseInt(metricsConf.get(HANDLER_THREAD_COUNT, "20"));
- }
- return 20;
- }
-
- public boolean isTimelineMetricsServiceWatcherDisabled() {
- if (metricsConf != null) {
- return Boolean.parseBoolean(metricsConf.get(WATCHER_DISABLED, "false"));
- }
- return false;
- }
-
- public int getTimelineMetricsServiceWatcherInitDelay() {
- if (metricsConf != null) {
- return Integer.parseInt(metricsConf.get(WATCHER_INITIAL_DELAY, "600"));
- }
- return 600;
- }
-
- public int getTimelineMetricsServiceWatcherDelay() {
- if (metricsConf != null) {
- return Integer.parseInt(metricsConf.get(WATCHER_DELAY, "30"));
- }
- return 30;
- }
-
- public int getTimelineMetricsServiceWatcherTimeout() {
- if (metricsConf != null) {
- return Integer.parseInt(metricsConf.get(WATCHER_TIMEOUT, "30"));
- }
- return 30;
- }
-
- public int getTimelineMetricsServiceWatcherMaxFailures() {
- if (metricsConf != null) {
- return Integer.parseInt(metricsConf.get(WATCHER_MAX_FAILURES, "3"));
- }
- return 3;
- }
-
- public boolean getTimelineMetricsMultipleClusterSupport() {
- if (metricsConf != null) {
- return Boolean.parseBoolean(metricsConf.get(TIMELINE_METRICS_SUPPORT_MULTIPLE_CLUSTERS, "false"));
- }
- return false;
- }
-
- public String getTimelineServiceRpcAddress() {
- String defaultRpcAddress = "0.0.0.0:60200";
- if (metricsConf != null) {
- return metricsConf.get(TIMELINE_SERVICE_RPC_ADDRESS, defaultRpcAddress);
- }
- return defaultRpcAddress;
- }
-
- public String getKafkaServers() {
- if (metricsConf != null) {
- return metricsConf.get("timeline.metrics.kafka.servers", null);
- }
- return null;
- }
-
- public boolean isDistributedCollectorModeDisabled() {
- try {
- if (getMetricsConf() != null) {
- return Boolean.parseBoolean(getMetricsConf().get("timeline.metrics.service.distributed.collector.mode.disabled", "false"));
- }
- return false;
- } catch (Exception e) {
- return false;
- }
- }
-
- public boolean isSecurityEnabled() {
- return hbaseConf.get("hbase.security.authentication", "").equals("kerberos");
- }
-
- public Set<String> getAmshbaseWhitelist() {
-
- ClassLoader classLoader = Thread.currentThread().getContextClassLoader();
- if (classLoader == null) {
- classLoader = getClass().getClassLoader();
- }
-
- BufferedReader br = null;
- String strLine;
- Set<String> whitelist = new HashSet<>();
-
- try(InputStream inputStream = classLoader.getResourceAsStream(AMSHBASE_METRICS_WHITESLIST_FILE)) {
-
- if (inputStream == null) {
- LOG.info("ams-hbase metrics whitelist file not present.");
- return Collections.EMPTY_SET;
- }
-
- br = new BufferedReader(new InputStreamReader(inputStream));
-
- while ((strLine = br.readLine()) != null) {
- strLine = strLine.trim();
- if (StringUtils.isEmpty(strLine)) {
- continue;
- }
- whitelist.add(strLine);
- }
- } catch (Exception ex) {
- LOG.error("Unable to read ams-hbase metric whitelist file", ex);
- return Collections.EMPTY_SET;
- }
-
- return whitelist;
- }
-
- /**
- * Get the sink interval for a metrics source.
- * Determines how often the metrics will be written to the sink.
- * This determines whether any caching will be needed on the collector
- * side, default interval disables caching by writing at the same time as
- * we get data.
- *
- * @param sinkProviderClassName Simple name of your implementation of {@link ExternalSinkProvider}
- * @param sourceName {@link SOURCE_NAME}
- * @return seconds
- */
- public int getExternalSinkInterval(String sinkProviderClassName,
- SOURCE_NAME sourceName) {
- String sinkProviderSimpleClassName = sinkProviderClassName.substring(
- sinkProviderClassName.lastIndexOf(".") + 1);
-
- return Integer.parseInt(metricsConf.get(
- String.format(EXTERNAL_SINK_INTERVAL, sinkProviderSimpleClassName, sourceName), "-1"));
- }
-
- public InternalSourceProvider getInternalSourceProvider() {
- Class<? extends InternalSourceProvider> providerClass =
- metricsConf.getClass(TIMELINE_METRICS_SOURCE_PROVIDER_CLASS,
- DefaultInternalMetricsSourceProvider.class, InternalSourceProvider.class);
- return ReflectionUtils.newInstance(providerClass, metricsConf);
- }
-
- /**
- * List of external sink provider classes. Comma-separated.
- */
- public List<ExternalSinkProvider> getExternalSinkProviderList() {
- Class<?>[] providerClasses = metricsConf.getClasses(TIMELINE_METRICS_SINK_PROVIDER_CLASS);
- List<ExternalSinkProvider> providerList = new ArrayList<>();
- if (providerClasses != null) {
- for (Class<?> providerClass : providerClasses) {
- providerList.add((ExternalSinkProvider) ReflectionUtils.newInstance(providerClass, metricsConf));
- }
- }
- return providerList;
- }
-
- public String getInternalCacheHeapPercent(String instanceName) {
- String heapPercent = metricsConf.get(String.format(INTERNAL_CACHE_HEAP_PERCENT, instanceName));
- if (StringUtils.isEmpty(heapPercent)) {
- return "5%";
- } else {
- return heapPercent.endsWith("%") ? heapPercent : heapPercent + "%";
- }
- }
-
- public String getDefaultMetricsSinkDir() {
- String dirPath = metricsConf.get(DEFAULT_EXTERNAL_SINK_DIR);
- if (dirPath == null) {
- // Only one logger at the time of writing
- Appender appender = (Appender) Logger.getRootLogger().getAllAppenders().nextElement();
- if (appender instanceof FileAppender) {
- File f = new File(((FileAppender) appender).getFile());
- if (f.exists()) {
- dirPath = f.getParent();
- } else {
- dirPath = "/tmp";
- }
- }
- }
-
- return dirPath;
- }
-
- public boolean isHostInMemoryAggregationEnabled() {
- if (metricsConf != null) {
- return Boolean.valueOf(metricsConf.get(TIMELINE_METRICS_HOST_INMEMORY_AGGREGATION, "false"));
- } else {
- return false;
- }
- }
-
- public boolean isContainerMetricsDisabled() {
- try {
- return metricsConf != null && Boolean.parseBoolean(metricsConf.get(TIMELINE_SERVICE_DISABLE_CONTAINER_METRICS, "false"));
- } catch (Exception e) {
-
- return false;
- }
- }
-
- public boolean isCollectorInMemoryAggregationEnabled() {
- if (metricsConf != null) {
- return Boolean.valueOf(metricsConf.get(TIMELINE_METRICS_COLLECTOR_INMEMORY_AGGREGATION, "false"));
- } else {
- return false;
- }
- }
-
- public List<String> getAppIdsForHostAggregation() {
- String appIds = metricsConf.get(CLUSTER_AGGREGATOR_APP_IDS);
- if (!StringUtils.isEmpty(appIds)) {
- return Arrays.asList(StringUtils.stripAll(appIds.split(",")));
- }
- return Collections.emptyList();
- }
-
- public String getZkConnectionUrl(String zkClientPort, String zkQuorum){
- StringBuilder sb = new StringBuilder();
- String[] quorumParts = zkQuorum.split(",");
- String prefix = "";
- for (String part : quorumParts) {
- sb.append(prefix);
- sb.append(part.trim());
- if (!part.contains(":")) {
- sb.append(":");
- sb.append(zkClientPort);
- }
- prefix = ",";
- }
-
- return sb.toString();
-
- }
-
- public boolean isWhitelistingEnabled() {
- if (metricsConf != null) {
- return Boolean.parseBoolean(metricsConf.get(TIMELINE_METRICS_WHITELIST_ENABLED, "false"));
- }
- return false;
- }
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricDistributedCache.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricDistributedCache.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricDistributedCache.java
deleted file mode 100644
index abedc7b..0000000
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricDistributedCache.java
+++ /dev/null
@@ -1,32 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline;
-
-import java.util.Collection;
-import java.util.Map;
-
-import org.apache.hadoop.metrics2.sink.timeline.MetricClusterAggregate;
-import org.apache.hadoop.metrics2.sink.timeline.TimelineMetric;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.aggregators.TimelineClusterMetric;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.discovery.TimelineMetricMetadataManager;
-
-public interface TimelineMetricDistributedCache {
- Map<TimelineClusterMetric, MetricClusterAggregate> evictMetricAggregates(Long startTime, Long endTime);
- void putMetrics(Collection<TimelineMetric> elements, TimelineMetricMetadataManager metricMetadataManager);
- Map<String, Double> getPointInTimeCacheMetrics();
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricStore.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricStore.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricStore.java
deleted file mode 100644
index 1a9c725..0000000
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricStore.java
+++ /dev/null
@@ -1,110 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline;
-
-import java.io.IOException;
-import java.sql.SQLException;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-import org.apache.hadoop.metrics2.sink.timeline.AggregationResult;
-import org.apache.hadoop.metrics2.sink.timeline.ContainerMetric;
-import org.apache.hadoop.metrics2.sink.timeline.Precision;
-import org.apache.hadoop.metrics2.sink.timeline.TimelineMetric;
-import org.apache.hadoop.metrics2.sink.timeline.TimelineMetricMetadata;
-import org.apache.hadoop.metrics2.sink.timeline.TimelineMetrics;
-import org.apache.hadoop.metrics2.sink.timeline.TopNConfig;
-import org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse;
-
-public interface TimelineMetricStore {
- /**
- * This method retrieves metrics stored by the Timeline store.
- *
- * @param metricNames Names of the metric, e.g.: cpu_user
- * @param hostnames Names of the host where the metric originated from
- * @param applicationId Id of the application to which this metric belongs
- * @param instanceId Application instance id.
- * @param startTime Start timestamp
- * @param endTime End timestamp
- * @param precision Precision [ seconds, minutes, hours ]
- * @param limit Override default result limit
- * @param groupedByHosts Group {@link TimelineMetric} by metric name, hostname,
- * app id and instance id
- * @param seriesAggregateFunction Specify this when caller want to aggregate multiple metrics
- * series into one. [ SUM, AVG, MIN, MAX ]
- *
- * @return {@link TimelineMetric}
- * @throws java.sql.SQLException
- */
- TimelineMetrics getTimelineMetrics(List<String> metricNames, List<String> hostnames,
- String applicationId, String instanceId, Long startTime,
- Long endTime, Precision precision, Integer limit, boolean groupedByHosts,
- TopNConfig topNConfig, String seriesAggregateFunction)
- throws SQLException, IOException;
-
- /**
- * Stores metric information to the timeline store. Any errors occurring for
- * individual put request objects will be reported in the response.
- *
- * @param metrics An {@link TimelineMetrics}.
- * @return An {@link org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse}.
- * @throws SQLException, IOException
- */
- TimelinePutResponse putMetrics(TimelineMetrics metrics) throws SQLException, IOException;
-
- /**
- * Store container metric into the timeline tore
- */
- TimelinePutResponse putContainerMetrics(List<ContainerMetric> metrics)
- throws SQLException, IOException;
-
- /**
- * Return all metrics metadata that have been written to the store.
- * @return { appId : [ @TimelineMetricMetadata ] }
- * @throws SQLException
- * @throws IOException
- */
- Map<String, List<TimelineMetricMetadata>> getTimelineMetricMetadata(String appId, String metricPattern,
- boolean includeBlacklistedMetrics) throws SQLException, IOException;
-
- TimelinePutResponse putHostAggregatedMetrics(AggregationResult aggregationResult) throws SQLException, IOException;
- /**
- * Returns all hosts that have written metrics with the apps on the host
- * @return { hostname : [ appIds ] }
- * @throws SQLException
- * @throws IOException
- */
- Map<String, Set<String>> getHostAppsMetadata() throws SQLException, IOException;
-
- /**
- * Returns all instances and the set of hosts each instance is present on
- * @return { instanceId : [ hosts ] }
- * @throws SQLException
- * @throws IOException
- */
- Map<String, Map<String,Set<String>>> getInstanceHostsMetadata(String instanceId, String appId) throws SQLException, IOException;
-
- byte[] getUuid(String metricName, String appId, String instanceId, String hostname) throws SQLException, IOException;
-
- /**
- * Return a list of known live collector nodes
- * @return [ hostname ]
- */
- List<String> getLiveInstances();
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricStoreWatcher.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricStoreWatcher.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricStoreWatcher.java
deleted file mode 100644
index a1ba5d7..0000000
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricStoreWatcher.java
+++ /dev/null
@@ -1,126 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline;
-
-import java.util.Collections;
-import java.util.concurrent.Callable;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.Future;
-import java.util.concurrent.TimeUnit;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.metrics2.sink.timeline.Precision;
-import org.apache.hadoop.metrics2.sink.timeline.TimelineMetric;
-import org.apache.hadoop.metrics2.sink.timeline.TimelineMetrics;
-import org.apache.hadoop.util.ExitUtil;
-
-/**
- * Acts as the single TimetineMetricStore Watcher.
- */
-public class TimelineMetricStoreWatcher implements Runnable {
-
- private static final Log LOG = LogFactory.getLog(TimelineMetricStoreWatcher.class);
- private static final String FAKE_METRIC_NAME = "TimelineMetricStoreWatcher.FakeMetric";
- private static final String FAKE_HOSTNAME = "fakehostname";
- private static final String FAKE_APP_ID = "timeline_metric_store_watcher";
-
- private static int failures = 0;
- private final TimelineMetricConfiguration configuration;
-
- private TimelineMetricStore timelineMetricStore;
-
- //used to call timelineMetricStore blocking methods with timeout
- private ExecutorService executor = Executors.newSingleThreadExecutor();
-
-
- public TimelineMetricStoreWatcher(TimelineMetricStore timelineMetricStore,
- TimelineMetricConfiguration configuration) {
- this.timelineMetricStore = timelineMetricStore;
- this.configuration = configuration;
- }
-
- @Override
- public void run() {
- if (checkMetricStore()) {
- failures = 0;
- if (LOG.isDebugEnabled()) {
- LOG.debug("Successfully got metrics from TimelineMetricStore");
- }
- } else {
- LOG.info("Failed to get metrics from TimelineMetricStore, attempt = " + failures);
- failures++;
- }
-
- if (failures >= configuration.getTimelineMetricsServiceWatcherMaxFailures()) {
- String msg = "Error getting metrics from TimelineMetricStore. " +
- "Shutting down by TimelineMetricStoreWatcher.";
- LOG.fatal(msg);
- ExitUtil.terminate(-1, msg);
- }
-
- }
-
- /**
- * Checks TimelineMetricStore functionality by adding and getting
- * a fake metric to/from HBase
- * @return if check was successful
- */
- private boolean checkMetricStore() {
- final long startTime = System.currentTimeMillis();
- final int delay = configuration.getTimelineMetricsServiceWatcherDelay();
- final int timeout = configuration.getTimelineMetricsServiceWatcherTimeout();
-
- TimelineMetric fakeMetric = new TimelineMetric();
- fakeMetric.setMetricName(FAKE_METRIC_NAME);
- fakeMetric.setHostName(FAKE_HOSTNAME);
- fakeMetric.setAppId(FAKE_APP_ID);
- fakeMetric.setStartTime(startTime);
- fakeMetric.getMetricValues().put(startTime, 0.0);
-
- final TimelineMetrics metrics = new TimelineMetrics();
- metrics.setMetrics(Collections.singletonList(fakeMetric));
-
- Callable<TimelineMetric> task = new Callable<TimelineMetric>() {
- public TimelineMetric call() throws Exception {
- timelineMetricStore.putMetrics(metrics);
- TimelineMetrics timelineMetrics = timelineMetricStore.getTimelineMetrics(
- Collections.singletonList(FAKE_METRIC_NAME), Collections.singletonList(FAKE_HOSTNAME),
- FAKE_APP_ID, null, startTime - delay * 2 * 1000,
- startTime + delay * 2 * 1000, Precision.SECONDS, 1, true, null, null);
- return timelineMetrics.getMetrics().get(0);
- }
- };
-
- Future<TimelineMetric> future = executor.submit(task);
- TimelineMetric timelineMetric = null;
- try {
- timelineMetric = future.get(timeout, TimeUnit.SECONDS);
- // Phoenix might throw RuntimeExeption's
- } catch (Exception e) {
- return false;
- } finally {
- future.cancel(true);
- }
-
- return timelineMetric != null;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricsAggregatorSink.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricsAggregatorSink.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricsAggregatorSink.java
deleted file mode 100644
index 7b03b30..0000000
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricsAggregatorSink.java
+++ /dev/null
@@ -1,60 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline;
-
-import java.util.Map;
-
-import org.apache.hadoop.metrics2.sink.timeline.MetricClusterAggregate;
-import org.apache.hadoop.metrics2.sink.timeline.MetricHostAggregate;
-import org.apache.hadoop.metrics2.sink.timeline.Precision;
-import org.apache.hadoop.metrics2.sink.timeline.TimelineMetric;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.aggregators.TimelineClusterMetric;
-
-/**
- * This Interface for storing aggregated metrics to any external storage
- */
-public interface TimelineMetricsAggregatorSink {
-
- /**
- * Save host aggregated metrics
- *
- * @param hostAggregateMap Map of host aggregated metrics
- * @param precision SECOND, MINUTE, HOUR, DAY
- */
- void saveHostAggregateRecords(
- Map<TimelineMetric, MetricHostAggregate> hostAggregateMap,
- Precision precision);
-
- /**
- * Save cluster time aggregated metrics
- *
- * @param clusterTimeAggregateMap Map of cluster aggregated metrics
- * @param precision SECOND, MINUTE, HOUR, DAY
- */
- void saveClusterTimeAggregateRecords(
- Map<TimelineClusterMetric, MetricHostAggregate> clusterTimeAggregateMap,
- Precision precision);
-
- /**
- * Save cluster aggregated metrics
- *
- * @param clusterAggregateMap Map of cluster aggregated metrics
- */
- void saveClusterAggregateRecords(
- Map<TimelineClusterMetric, MetricClusterAggregate> clusterAggregateMap);
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricsFilter.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricsFilter.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricsFilter.java
deleted file mode 100644
index ef7186c..0000000
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricsFilter.java
+++ /dev/null
@@ -1,170 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline;
-
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.TIMELINE_METRICS_APPS_BLACKLIST;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.TIMELINE_METRICS_APPS_WHITELIST;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.TIMELINE_METRICS_WHITELIST_FILE;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.TIMELINE_METRICS_WHITELIST_FILE_LOCATION_DEFAULT;
-
-import java.io.BufferedReader;
-import java.io.FileInputStream;
-import java.io.IOException;
-import java.io.InputStreamReader;
-import java.util.Arrays;
-import java.util.HashSet;
-import java.util.Set;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-import org.apache.commons.collections.CollectionUtils;
-import org.apache.commons.lang.StringUtils;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.metrics2.sink.timeline.TimelineMetric;
-
-public class TimelineMetricsFilter {
-
- private static Set<String> whitelistedMetrics;
- private static Set<Pattern> whitelistedMetricPatterns;
- private static Set<String> whitelistedApps;
- private static Set<String> blacklistedApps;
- private static String patternPrefix = "._p_";
- private static Set<String> amshbaseWhitelist;
-
- private static final Log LOG = LogFactory.getLog(TimelineMetricsFilter.class);
-
- public static void initializeMetricFilter(TimelineMetricConfiguration configuration) {
-
- Configuration metricsConf = null;
- try {
- metricsConf = configuration.getMetricsConf();
- } catch (Exception e) {
- LOG.error("Error fetching metrics configuration for getting whitelisting information");
- return;
- }
-
- whitelistedMetrics = new HashSet<String>();
- whitelistedMetricPatterns = new HashSet<Pattern>();
- blacklistedApps = new HashSet<>();
- whitelistedApps = new HashSet<>();
- amshbaseWhitelist = new HashSet<>();
-
- if (configuration.isWhitelistingEnabled()) {
- String whitelistFile = metricsConf.get(TIMELINE_METRICS_WHITELIST_FILE, TIMELINE_METRICS_WHITELIST_FILE_LOCATION_DEFAULT);
- readMetricWhitelistFromFile(whitelistFile);
- }
-
- String appsBlacklist = metricsConf.get(TIMELINE_METRICS_APPS_BLACKLIST, "");
- if (!StringUtils.isEmpty(appsBlacklist)) {
- for (String app : appsBlacklist.split(",")) {
- blacklistedApps.add(app);
- }
- LOG.info("Blacklisted apps : " + blacklistedApps.toString());
- }
-
- String appsWhitelist = metricsConf.get(TIMELINE_METRICS_APPS_WHITELIST, "");
- if (!StringUtils.isEmpty(appsWhitelist)) {
- for (String app : appsWhitelist.split(",")) {
- whitelistedApps.add(app);
- }
- LOG.info("Whitelisted apps : " + whitelistedApps.toString());
- }
-
- amshbaseWhitelist = configuration.getAmshbaseWhitelist();
- if (CollectionUtils.isNotEmpty(amshbaseWhitelist)) {
- LOG.info("Whitelisting " + amshbaseWhitelist.size() + " ams-hbase metrics");
- }
- }
-
- private static void readMetricWhitelistFromFile(String whitelistFile) {
-
- BufferedReader br = null;
- String strLine;
-
- try(FileInputStream fstream = new FileInputStream(whitelistFile)) {
- br = new BufferedReader(new InputStreamReader(fstream));
-
- while ((strLine = br.readLine()) != null) {
- strLine = strLine.trim();
- if (StringUtils.isEmpty(strLine)) {
- continue;
- }
- if (strLine.startsWith(patternPrefix)) {
- whitelistedMetricPatterns.add(Pattern.compile(strLine.substring(patternPrefix.length())));
- } else {
- whitelistedMetrics.add(strLine);
- }
- }
- } catch (IOException ioEx) {
- LOG.error("Unable to parse metric whitelist file", ioEx);
- }
-
- LOG.info("Whitelisting " + whitelistedMetrics.size() + " metrics");
- LOG.debug("Whitelisted metrics : " + Arrays.toString(whitelistedMetrics.toArray()));
- }
-
- public static boolean acceptMetric(String metricName, String appId) {
- TimelineMetric timelineMetric = new TimelineMetric();
- timelineMetric.setAppId(appId);
- timelineMetric.setMetricName(metricName);
- return acceptMetric(timelineMetric);
- }
-
- public static boolean acceptMetric(TimelineMetric metric) {
-
- String appId = metric.getAppId();
- String metricName = metric.getMetricName();
- // App Blacklisting
- if (CollectionUtils.isNotEmpty(blacklistedApps) && blacklistedApps.contains(appId)) {
- return false;
- }
-
- //Special Case appId = ams-hbase whitelisting.
- if ("ams-hbase".equals(appId) && CollectionUtils.isNotEmpty(amshbaseWhitelist)) {
- return amshbaseWhitelist.contains(metric.getMetricName());
- }
-
- // App Whitelisting
- if (CollectionUtils.isNotEmpty(whitelistedApps) && whitelistedApps.contains(appId)) {
- return true;
- }
-
- // Metric Whitelisting
- if (CollectionUtils.isEmpty(whitelistedMetrics) && CollectionUtils.isEmpty(whitelistedMetricPatterns)) {
- return true;
- }
-
- if (whitelistedMetrics.contains(metricName)) {
- return true;
- }
-
- for (Pattern p : whitelistedMetricPatterns) {
- Matcher m = p.matcher(metricName);
- if (m.find()) {
- whitelistedMetrics.add(metricName);
- return true;
- }
- }
-
- return false;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricsIgniteCache.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricsIgniteCache.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricsIgniteCache.java
deleted file mode 100644
index 24fc938..0000000
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricsIgniteCache.java
+++ /dev/null
@@ -1,296 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline;
-
-import static java.util.concurrent.TimeUnit.SECONDS;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.CLUSTER_AGGREGATOR_SECOND_SLEEP_INTERVAL;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.CLUSTER_AGGREGATOR_TIMESLICE_INTERVAL;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.HOST_APP_ID;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.TIMELINE_METRICS_CLUSTER_AGGREGATOR_INTERPOLATION_ENABLED;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.TIMELINE_METRICS_COLLECTOR_IGNITE_BACKUPS;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.TIMELINE_METRICS_COLLECTOR_IGNITE_NODES;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.TIMELINE_METRIC_AGGREGATION_SQL_FILTERS;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.TIMELINE_SERVICE_HTTP_POLICY;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.aggregators.AggregatorUtils.getRoundedCheckPointTimeMillis;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.aggregators.AggregatorUtils.getTimeSlices;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.aggregators.AggregatorUtils.sliceFromTimelineMetric;
-
-import java.net.MalformedURLException;
-import java.net.URISyntaxException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.locks.Lock;
-
-import javax.cache.Cache;
-import javax.cache.expiry.CreatedExpiryPolicy;
-import javax.cache.expiry.Duration;
-
-import org.apache.commons.lang.StringUtils;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.metrics2.sink.timeline.MetricClusterAggregate;
-import org.apache.hadoop.metrics2.sink.timeline.TimelineMetric;
-import org.apache.hadoop.metrics2.sink.timeline.availability.MetricCollectorHAHelper;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.aggregators.TimelineClusterMetric;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.discovery.TimelineMetricHostMetadata;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.discovery.TimelineMetricMetadataManager;
-import org.apache.ignite.Ignite;
-import org.apache.ignite.IgniteCache;
-import org.apache.ignite.Ignition;
-import org.apache.ignite.cache.CacheAtomicityMode;
-import org.apache.ignite.cache.CacheMetrics;
-import org.apache.ignite.cache.CacheMode;
-import org.apache.ignite.cache.query.QueryCursor;
-import org.apache.ignite.cache.query.ScanQuery;
-import org.apache.ignite.configuration.CacheConfiguration;
-import org.apache.ignite.configuration.IgniteConfiguration;
-import org.apache.ignite.lang.IgniteBiPredicate;
-import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
-import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
-import org.apache.ignite.ssl.SslContextFactory;
-
-public class TimelineMetricsIgniteCache implements TimelineMetricDistributedCache {
- private static final Log LOG =
- LogFactory.getLog(TimelineMetricsIgniteCache.class);
- private IgniteCache<TimelineClusterMetric, MetricClusterAggregate> igniteCache;
- private long cacheSliceIntervalMillis;
- private boolean interpolationEnabled;
- private List<String> skipAggrPatternStrings = new ArrayList<>();
- private List<String> appIdsToAggregate;
-
-
- public TimelineMetricsIgniteCache() throws MalformedURLException, URISyntaxException {
- TimelineMetricConfiguration timelineMetricConfiguration = TimelineMetricConfiguration.getInstance();
- Configuration metricConf = timelineMetricConfiguration.getMetricsConf();
- Configuration sslConf = timelineMetricConfiguration.getMetricsSslConf();
-
- IgniteConfiguration igniteConfiguration = new IgniteConfiguration();
-
- //TODO add config to disable logging
-
- //enable ssl for ignite requests
- if (metricConf.get(TIMELINE_SERVICE_HTTP_POLICY) != null && metricConf.get(TIMELINE_SERVICE_HTTP_POLICY).equalsIgnoreCase("HTTPS_ONLY")) {
- SslContextFactory sslContextFactory = new SslContextFactory();
- String keyStorePath = sslConf.get("ssl.server.keystore.location");
- String keyStorePassword = sslConf.get("ssl.server.keystore.password");
- String trustStorePath = sslConf.get("ssl.server.truststore.location");
- String trustStorePassword = sslConf.get("ssl.server.truststore.password");
-
- sslContextFactory.setKeyStoreFilePath(keyStorePath);
- sslContextFactory.setKeyStorePassword(keyStorePassword.toCharArray());
- sslContextFactory.setTrustStoreFilePath(trustStorePath);
- sslContextFactory.setTrustStorePassword(trustStorePassword.toCharArray());
- igniteConfiguration.setSslContextFactory(sslContextFactory);
- }
-
- //aggregation parameters
- appIdsToAggregate = timelineMetricConfiguration.getAppIdsForHostAggregation();
- interpolationEnabled = Boolean.parseBoolean(metricConf.get(TIMELINE_METRICS_CLUSTER_AGGREGATOR_INTERPOLATION_ENABLED, "true"));
- cacheSliceIntervalMillis = SECONDS.toMillis(metricConf.getInt(CLUSTER_AGGREGATOR_TIMESLICE_INTERVAL, 30));
- Long aggregationInterval = metricConf.getLong(CLUSTER_AGGREGATOR_SECOND_SLEEP_INTERVAL, 120L);
-
- String filteredMetricPatterns = metricConf.get(TIMELINE_METRIC_AGGREGATION_SQL_FILTERS);
- if (!StringUtils.isEmpty(filteredMetricPatterns)) {
- LOG.info("Skipping aggregation for metric patterns : " + filteredMetricPatterns);
- skipAggrPatternStrings.addAll(Arrays.asList(filteredMetricPatterns.split(",")));
- }
-
- if (metricConf.get(TIMELINE_METRICS_COLLECTOR_IGNITE_NODES) != null) {
- TcpDiscoverySpi discoverySpi = new TcpDiscoverySpi();
- TcpDiscoveryVmIpFinder ipFinder = new TcpDiscoveryVmIpFinder();
- ipFinder.setAddresses(Arrays.asList(metricConf.get(TIMELINE_METRICS_COLLECTOR_IGNITE_NODES).split(",")));
- LOG.info("Setting ignite nodes to : " + ipFinder.getRegisteredAddresses());
- discoverySpi.setIpFinder(ipFinder);
- igniteConfiguration.setDiscoverySpi(discoverySpi);
- } else {
- //get live nodes from ZK
- String zkClientPort = timelineMetricConfiguration.getClusterZKClientPort();
- String zkQuorum = timelineMetricConfiguration.getClusterZKQuorum();
- String zkConnectionURL = timelineMetricConfiguration.getZkConnectionUrl(zkClientPort, zkQuorum);
- MetricCollectorHAHelper metricCollectorHAHelper = new MetricCollectorHAHelper(zkConnectionURL, 5, 200);
- Collection<String> liveCollectors = metricCollectorHAHelper.findLiveCollectorHostsFromZNode();
- if (liveCollectors != null && !liveCollectors.isEmpty()) {
- TcpDiscoverySpi discoverySpi = new TcpDiscoverySpi();
- TcpDiscoveryVmIpFinder ipFinder = new TcpDiscoveryVmIpFinder();
- ipFinder.setAddresses(liveCollectors);
- LOG.info("Setting ignite nodes to : " + ipFinder.getRegisteredAddresses());
- discoverySpi.setIpFinder(ipFinder);
- igniteConfiguration.setDiscoverySpi(discoverySpi);
- }
- }
-
-
- //ignite cache configuration
- CacheConfiguration<TimelineClusterMetric, MetricClusterAggregate> cacheConfiguration = new CacheConfiguration<>();
- cacheConfiguration.setName("metrics_cache");
- //set cache mode to partitioned with # of backups
- cacheConfiguration.setCacheMode(CacheMode.PARTITIONED);
- cacheConfiguration.setBackups(metricConf.getInt(TIMELINE_METRICS_COLLECTOR_IGNITE_BACKUPS, 1));
- //disable throttling due to cpu impact
- cacheConfiguration.setRebalanceThrottle(0);
- //enable locks
- cacheConfiguration.setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL);
- //expiry policy to remove lost keys, if any
- cacheConfiguration.setEagerTtl(true);
- cacheConfiguration.setExpiryPolicyFactory(CreatedExpiryPolicy.factoryOf(new Duration(TimeUnit.SECONDS, aggregationInterval * 3)));
-
- Ignite igniteNode = Ignition.start(igniteConfiguration);
- igniteCache = igniteNode.getOrCreateCache(cacheConfiguration);
- }
-
- /**
- * Looks through the cache and evicts all elements within (startTime; endTime] half-interval
- * All elements satisfying the half-interval will be removed from the cache.
- * @param startTime
- * @param endTime
- * @return
- */
- @Override
- public Map<TimelineClusterMetric, MetricClusterAggregate> evictMetricAggregates(Long startTime, Long endTime) {
- Map<TimelineClusterMetric, MetricClusterAggregate> aggregatedMetricsMap = new HashMap<>();
-
- //construct filter
- IgniteBiPredicate<TimelineClusterMetric, MetricClusterAggregate> filter =
- (IgniteBiPredicate<TimelineClusterMetric, MetricClusterAggregate>) (key, value) -> key.getTimestamp() > startTime && key.getTimestamp() <= endTime;
-
- //get values from cache
- try (QueryCursor<Cache.Entry<TimelineClusterMetric, MetricClusterAggregate>> cursor = igniteCache.query(new ScanQuery(filter))) {
- for (Cache.Entry<TimelineClusterMetric, MetricClusterAggregate> e : cursor) {
- aggregatedMetricsMap.put(e.getKey(), e.getValue());
- }
- }
-
- //remove values from cache
- igniteCache.removeAllAsync(aggregatedMetricsMap.keySet());
-
- return aggregatedMetricsMap;
- }
-
- /**
- * Iterates through elements skipping white-listed patterns;
- * calculates average value for each slice of each metric (last slice values could be ignored in there is the possibility that values from this slice could be present in next post);
- * updates/adds the value in the cache;
- * calculates applications host metrics based on the metadata of hosted apps
- * updates metadata of hosted apps if needed
- * @param elements
- * @param metadataManager
- */
- @Override
- public void putMetrics(Collection<TimelineMetric> elements, TimelineMetricMetadataManager metadataManager) {
- Map<String, TimelineMetricHostMetadata> hostMetadata = metadataManager.getHostedAppsCache();
- for (TimelineMetric metric : elements) {
- if (shouldBeSkipped(metric.getMetricName())) {
- if (LOG.isDebugEnabled()) {
- LOG.debug(String.format("Skipping %s metric from being aggregated", metric.getMetricName()));
- }
- continue;
- }
- List<Long[]> timeSlices = getTimeSlices(getRoundedCheckPointTimeMillis(metric.getMetricValues().firstKey(), cacheSliceIntervalMillis), metric.getMetricValues().lastKey(), cacheSliceIntervalMillis);
- Map<TimelineClusterMetric, Double> slicedClusterMetrics = sliceFromTimelineMetric(metric, timeSlices, interpolationEnabled);
-
- if (slicedClusterMetrics != null) {
- for (Map.Entry<TimelineClusterMetric, Double> metricDoubleEntry : slicedClusterMetrics.entrySet()) {
- MetricClusterAggregate newMetricClusterAggregate = new MetricClusterAggregate(
- metricDoubleEntry.getValue(), 1, null, metricDoubleEntry.getValue(), metricDoubleEntry.getValue());
- //put app metric into cache
- putMetricIntoCache(metricDoubleEntry.getKey(), newMetricClusterAggregate);
- if (hostMetadata != null) {
- //calculate app host metric
- if (metric.getAppId().equalsIgnoreCase(HOST_APP_ID)) {
- // Candidate metric, update app aggregates
- if (hostMetadata.containsKey(metric.getHostName())) {
- updateAppAggregatesFromHostMetric(metricDoubleEntry.getKey(), newMetricClusterAggregate, hostMetadata.get(metric.getHostName()));
- }
- } else {
- // Build the hostedapps map if not a host metric
- // Check app candidacy for host aggregation
- //TODO better to lock TimelineMetricHostMetadata instance to avoid dataloss, but generally the data could be lost only during initial collector start
- if (appIdsToAggregate.contains(metric.getAppId())) {
- TimelineMetricHostMetadata timelineMetricHostMetadata = hostMetadata.get(metric.getHostName());
- ConcurrentHashMap<String, String> appIdsMap;
- if (timelineMetricHostMetadata == null) {
- appIdsMap = new ConcurrentHashMap<>();
- hostMetadata.put(metric.getHostName(), new TimelineMetricHostMetadata(appIdsMap));
- } else {
- appIdsMap = timelineMetricHostMetadata.getHostedApps();
- }
- if (!appIdsMap.containsKey(metric.getAppId())) {
- appIdsMap.put(metric.getAppId(), metric.getAppId());
- LOG.info("Adding appId to hosted apps: appId = " +
- metric.getAppId() + ", hostname = " + metric.getHostName());
- }
- }
- }
- }
- }
- }
- }
- }
-
- private void updateAppAggregatesFromHostMetric(TimelineClusterMetric key, MetricClusterAggregate newMetricClusterAggregate, TimelineMetricHostMetadata timelineMetricHostMetadata) {
- for (String appId : timelineMetricHostMetadata.getHostedApps().keySet()) {
- TimelineClusterMetric timelineClusterMetric = new TimelineClusterMetric(key.getMetricName(), appId, key.getInstanceId(), key.getTimestamp());
- putMetricIntoCache(timelineClusterMetric, newMetricClusterAggregate);
- }
- }
-
- private void putMetricIntoCache(TimelineClusterMetric metricKey, MetricClusterAggregate metricValue) {
- Lock lock = igniteCache.lock(metricKey);
- lock.lock();
- try {
- MetricClusterAggregate metricClusterAggregateFromCache = igniteCache.get(metricKey);
- if (metricClusterAggregateFromCache == null) {
- igniteCache.put(metricKey, metricValue);
- } else {
- metricClusterAggregateFromCache.updateAggregates(metricValue);
- igniteCache.put(metricKey, metricClusterAggregateFromCache);
- }
- } catch (Exception e) {
- LOG.error("Exception : ", e);
- } finally {
- lock.unlock();
- }
- }
-
- @Override
- public Map<String, Double> getPointInTimeCacheMetrics() {
- CacheMetrics clusterIgniteMetrics = igniteCache.metrics();
- Map<String, Double> metricsMap = new HashMap<>();
- metricsMap.put("Cluster_AverageGetTime", (double) clusterIgniteMetrics.getAverageGetTime());
- metricsMap.put("Cluster_AveragePutTime", (double) clusterIgniteMetrics.getAveragePutTime());
- metricsMap.put("Cluster_KeySize", (double) clusterIgniteMetrics.getKeySize());
- metricsMap.put("Cluster_OffHeapAllocatedSize", (double) clusterIgniteMetrics.getOffHeapAllocatedSize());
- return metricsMap;
- }
-
- private boolean shouldBeSkipped(String metricName) {
- for (String pattern : skipAggrPatternStrings) {
- if (metricName.matches(pattern)) {
- return true;
- }
- }
- return false;
- }
-}
[15/28] ambari git commit: AMBARI-22740 : Fix integration test for
HBase in branch-3.0-ams due to UUID changes. (avijayan)
Posted by av...@apache.org.
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/AbstractTimelineAggregator.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/AbstractTimelineAggregator.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/AbstractTimelineAggregator.java
deleted file mode 100644
index 9cfba6e..0000000
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/AbstractTimelineAggregator.java
+++ /dev/null
@@ -1,495 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.aggregators;
-
-import static java.util.concurrent.TimeUnit.SECONDS;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.AGGREGATOR_CHECKPOINT_DELAY;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.RESULTSET_FETCH_SIZE;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.aggregators.AggregatorUtils.getRoundedAggregateTimeMillis;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.aggregators.AggregatorUtils.getRoundedCheckPointTimeMillis;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.availability.AggregationTaskRunner.ACTUAL_AGGREGATOR_NAMES;
-
-import java.io.File;
-import java.io.IOException;
-import java.sql.Connection;
-import java.sql.PreparedStatement;
-import java.sql.ResultSet;
-import java.sql.SQLException;
-import java.util.Date;
-import java.util.Iterator;
-import java.util.List;
-
-import org.apache.commons.collections.CollectionUtils;
-import org.apache.commons.io.FileUtils;
-import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.PhoenixHBaseAccessor;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.availability.AggregationTaskRunner;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.availability.AggregationTaskRunner.AGGREGATOR_NAME;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.availability.MetricCollectorHAController;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query.Condition;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query.EmptyCondition;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query.PhoenixTransactSQL;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Base class for all runnable aggregators. Provides common functions like
- * check pointing and scheduling.
- */
-public abstract class AbstractTimelineAggregator implements TimelineMetricAggregator {
- protected final PhoenixHBaseAccessor hBaseAccessor;
- protected final Logger LOG;
- protected final long checkpointDelayMillis;
- protected final Integer resultsetFetchSize;
- protected Configuration metricsConf;
- private String checkpointLocation;
- private Long sleepIntervalMillis;
- private Integer checkpointCutOffMultiplier;
- private String aggregatorDisableParam;
- protected String tableName;
- protected String outputTableName;
- protected Long nativeTimeRangeDelay;
- protected AggregationTaskRunner taskRunner;
- protected List<String> downsampleMetricPatterns;
- protected List<CustomDownSampler> configuredDownSamplers;
-
- // Explicitly name aggregators for logging needs
- private final AGGREGATOR_NAME aggregatorName;
-
- AbstractTimelineAggregator(AGGREGATOR_NAME aggregatorName,
- PhoenixHBaseAccessor hBaseAccessor,
- Configuration metricsConf) {
- this.aggregatorName = aggregatorName;
- this.hBaseAccessor = hBaseAccessor;
- this.metricsConf = metricsConf;
- this.checkpointDelayMillis = SECONDS.toMillis(metricsConf.getInt(AGGREGATOR_CHECKPOINT_DELAY, 120));
- this.resultsetFetchSize = metricsConf.getInt(RESULTSET_FETCH_SIZE, 2000);
- this.LOG = LoggerFactory.getLogger(ACTUAL_AGGREGATOR_NAMES.get(aggregatorName));
- this.configuredDownSamplers = DownSamplerUtils.getDownSamplers(metricsConf);
- this.downsampleMetricPatterns = DownSamplerUtils.getDownsampleMetricPatterns(metricsConf);
- }
-
- public AbstractTimelineAggregator(AGGREGATOR_NAME aggregatorName,
- PhoenixHBaseAccessor hBaseAccessor,
- Configuration metricsConf,
- String checkpointLocation,
- Long sleepIntervalMillis,
- Integer checkpointCutOffMultiplier,
- String aggregatorDisableParam,
- String tableName,
- String outputTableName,
- Long nativeTimeRangeDelay,
- MetricCollectorHAController haController) {
- this(aggregatorName, hBaseAccessor, metricsConf);
- this.checkpointLocation = checkpointLocation;
- this.sleepIntervalMillis = sleepIntervalMillis;
- this.checkpointCutOffMultiplier = checkpointCutOffMultiplier;
- this.aggregatorDisableParam = aggregatorDisableParam;
- this.tableName = tableName;
- this.outputTableName = outputTableName;
- this.nativeTimeRangeDelay = nativeTimeRangeDelay;
- this.taskRunner = haController != null && haController.isInitialized() ?
- haController.getAggregationTaskRunner() : null;
- }
-
- @Override
- public void run() {
- LOG.info("Started Timeline aggregator thread @ " + new Date());
- Long SLEEP_INTERVAL = getSleepIntervalMillis();
- runOnce(SLEEP_INTERVAL);
- }
-
- /**
- * Access relaxed for tests
- */
- public void runOnce(Long SLEEP_INTERVAL) {
- boolean performAggregationFunction = true;
- if (taskRunner != null) {
- switch (getAggregatorType()) {
- case HOST:
- performAggregationFunction = taskRunner.performsHostAggregation();
- break;
- case CLUSTER:
- performAggregationFunction = taskRunner.performsClusterAggregation();
- }
- }
-
- if (performAggregationFunction) {
- long currentTime = System.currentTimeMillis();
- long lastCheckPointTime = readLastCheckpointSavingOnFirstRun(currentTime);
-
- if (lastCheckPointTime != -1) {
- LOG.info("Last check point time: " + lastCheckPointTime + ", lagBy: "
- + ((currentTime - lastCheckPointTime) / 1000)
- + " seconds.");
-
- boolean success = doWork(lastCheckPointTime, lastCheckPointTime + SLEEP_INTERVAL);
-
- if (success) {
- try {
- saveCheckPoint(lastCheckPointTime + SLEEP_INTERVAL);
- } catch (IOException io) {
- LOG.warn("Error saving checkpoint, restarting aggregation at " +
- "previous checkpoint.");
- }
- }
- }
- } else {
- LOG.info("Skipping aggregation function not owned by this instance.");
- }
- }
-
- private long readLastCheckpointSavingOnFirstRun(long currentTime) {
- long lastCheckPointTime = -1;
-
- try {
- lastCheckPointTime = readCheckPoint();
- if (lastCheckPointTime != -1) {
- LOG.info("Last Checkpoint read : " + new Date(lastCheckPointTime));
- if (isLastCheckPointTooOld(currentTime, lastCheckPointTime)) {
- LOG.warn("Last Checkpoint is too old, discarding last checkpoint. " +
- "lastCheckPointTime = " + new Date(lastCheckPointTime));
- lastCheckPointTime = getRoundedAggregateTimeMillis(getSleepIntervalMillis()) - getSleepIntervalMillis();
- LOG.info("Saving checkpoint time. " + new Date((lastCheckPointTime)));
- saveCheckPoint(lastCheckPointTime);
-
- } else {
-
- if (lastCheckPointTime > 0) {
- lastCheckPointTime = getRoundedCheckPointTimeMillis(lastCheckPointTime, getSleepIntervalMillis());
- LOG.info("Rounded off checkpoint : " + new Date(lastCheckPointTime));
- }
-
- if (isLastCheckPointTooYoung(lastCheckPointTime)) {
- LOG.info("Last checkpoint too recent for aggregation. Sleeping for 1 cycle.");
- return -1; //Skip Aggregation this time around
- }
- }
- } else {
- /*
- No checkpoint. Save current rounded checkpoint and sleep for 1 cycle.
- */
- LOG.info("No checkpoint found");
- long firstCheckPoint = getRoundedAggregateTimeMillis(getSleepIntervalMillis());
- LOG.info("Saving checkpoint time. " + new Date((firstCheckPoint)));
- saveCheckPoint(firstCheckPoint);
- }
- } catch (IOException io) {
- LOG.warn("Unable to write last checkpoint time. Resuming sleep.", io);
- }
- return lastCheckPointTime;
- }
-
- private boolean isLastCheckPointTooOld(long currentTime, long checkpoint) {
- // first checkpoint is saved checkpointDelayMillis in the past,
- // so here we also need to take it into account
- return checkpoint != -1 &&
- ((currentTime - checkpoint) > getCheckpointCutOffIntervalMillis());
- }
-
- private boolean isLastCheckPointTooYoung(long checkpoint) {
- return checkpoint != -1 &&
- ((getRoundedAggregateTimeMillis(getSleepIntervalMillis()) <= checkpoint));
- }
-
- protected long readCheckPoint() {
- if (taskRunner != null) {
- return taskRunner.getCheckpointManager().readCheckpoint(aggregatorName);
- }
- try {
- File checkpoint = new File(getCheckpointLocation());
- if (checkpoint.exists()) {
- String contents = FileUtils.readFileToString(checkpoint);
- if (contents != null && !contents.isEmpty()) {
- return Long.parseLong(contents);
- }
- }
- } catch (IOException io) {
- LOG.debug("", io);
- }
- return -1;
- }
-
- protected void saveCheckPoint(long checkpointTime) throws IOException {
- if (taskRunner != null) {
- boolean success = taskRunner.getCheckpointManager().writeCheckpoint(aggregatorName, checkpointTime);
- if (!success) {
- LOG.error("Error saving checkpoint with AggregationTaskRunner, " +
- "aggregator = " + aggregatorName + "value = " + checkpointTime);
- }
- } else {
- File checkpoint = new File(getCheckpointLocation());
- if (!checkpoint.exists()) {
- boolean done = checkpoint.createNewFile();
- if (!done) {
- throw new IOException("Could not create checkpoint at location, " +
- getCheckpointLocation());
- }
- }
- FileUtils.writeStringToFile(checkpoint, String.valueOf(checkpointTime));
- }
- }
-
- /**
- * Read metrics written during the time interval and save the sum and total
- * in the aggregate table.
- *
- * @param startTime Sample start time
- * @param endTime Sample end time
- */
- public boolean doWork(long startTime, long endTime) {
- LOG.info("Start aggregation cycle @ " + new Date() + ", " +
- "startTime = " + new Date(startTime) + ", endTime = " + new Date(endTime));
-
- boolean success = true;
- Condition condition = prepareMetricQueryCondition(startTime, endTime);
-
- Connection conn = null;
- PreparedStatement stmt = null;
- ResultSet rs = null;
-
- try {
- conn = hBaseAccessor.getConnection();
- // FLUME 2. aggregate and ignore the instance
- stmt = PhoenixTransactSQL.prepareGetMetricsSqlStmt(conn, condition);
-
- LOG.debug("Query issued @: " + new Date());
- if (condition.doUpdate()) {
- int rows = stmt.executeUpdate();
- conn.commit();
- LOG.info(rows + " row(s) updated in aggregation.");
-
- //TODO : Fix downsampling after UUID change.
- //downsample(conn, startTime, endTime);
- } else {
- rs = stmt.executeQuery();
- }
- LOG.debug("Query returned @: " + new Date());
-
- aggregate(rs, startTime, endTime);
-
- } catch (Exception e) {
- LOG.error("Exception during aggregating metrics.", e);
- success = false;
- } finally {
- if (rs != null) {
- try {
- rs.close();
- } catch (SQLException e) {
- // Ignore
- }
- }
- if (stmt != null) {
- try {
- stmt.close();
- } catch (SQLException e) {
- // Ignore
- }
- }
- if (conn != null) {
- try {
- conn.close();
- } catch (SQLException sql) {
- // Ignore
- }
- }
- }
-
- LOG.info("End aggregation cycle @ " + new Date());
- return success;
- }
-
- protected abstract Condition prepareMetricQueryCondition(long startTime, long endTime);
-
- protected abstract void aggregate(ResultSet rs, long startTime, long endTime) throws IOException, SQLException;
-
- protected void downsample(Connection conn, Long startTime, Long endTime) {
-
- LOG.debug("Checking for downsampling requests.");
- if (CollectionUtils.isEmpty(configuredDownSamplers)) {
- LOG.debug("No downsamplers configured");
- return;
- }
-
- // Generate UPSERT query prefix. UPSERT part of the query is needed on the Aggregator side.
- // SELECT part of the query is provided by the downsampler.
- String queryPrefix = PhoenixTransactSQL.DOWNSAMPLE_CLUSTER_METRIC_SQL_UPSERT_PREFIX;
- if (outputTableName.contains("RECORD")) {
- queryPrefix = PhoenixTransactSQL.DOWNSAMPLE_HOST_METRIC_SQL_UPSERT_PREFIX;
- }
- queryPrefix = String.format(queryPrefix, outputTableName);
-
- for (Iterator<CustomDownSampler> iterator = configuredDownSamplers.iterator(); iterator.hasNext();){
- CustomDownSampler downSampler = iterator.next();
-
- if (downSampler.validateConfigs()) {
- EmptyCondition downSamplingCondition = new EmptyCondition();
- downSamplingCondition.setDoUpdate(true);
- List<String> stmts = downSampler.prepareDownSamplingStatement(startTime, endTime, tableName);
- for (String stmt : stmts) {
- downSamplingCondition.setStatement(queryPrefix + stmt);
- runDownSamplerQuery(conn, downSamplingCondition);
- }
- } else {
- LOG.warn("The following downsampler failed config validation : " + downSampler.getClass().getName() + "." +
- "Removing it from downsamplers list.");
- iterator.remove();
- }
- }
-
- }
-
- public Long getSleepIntervalMillis() {
- return sleepIntervalMillis;
- }
-
- public void setSleepIntervalMillis(Long sleepIntervalMillis) {
- this.sleepIntervalMillis = sleepIntervalMillis;
- }
-
- protected Integer getCheckpointCutOffMultiplier() {
- return checkpointCutOffMultiplier;
- }
-
- protected Long getCheckpointCutOffIntervalMillis() {
- return getCheckpointCutOffMultiplier() * getSleepIntervalMillis();
- }
-
- public boolean isDisabled() {
- return metricsConf.getBoolean(aggregatorDisableParam, false);
- }
-
- protected String getQueryHint(Long startTime) {
- StringBuilder sb = new StringBuilder();
- sb.append("/*+ ");
- sb.append("NATIVE_TIME_RANGE(");
- sb.append(startTime - nativeTimeRangeDelay);
- sb.append(") ");
- if (hBaseAccessor.isSkipBlockCacheForAggregatorsEnabled()) {
- sb.append("NO_CACHE ");
- }
- sb.append("*/");
- return sb.toString();
- }
-
- protected String getCheckpointLocation() {
- return checkpointLocation;
- }
-
- /**
- * Run 1 downsampler query.
- * @param conn
- * @param condition
- */
- private void runDownSamplerQuery(Connection conn, Condition condition) {
-
- PreparedStatement stmt = null;
- ResultSet rs = null;
- LOG.debug("Downsampling query : " + condition.getStatement());
-
- try {
- stmt = PhoenixTransactSQL.prepareGetMetricsSqlStmt(conn, condition);
-
- LOG.debug("Downsampler Query issued...");
- if (condition.doUpdate()) {
- int rows = stmt.executeUpdate();
- conn.commit();
- LOG.info(rows + " row(s) updated in downsampling.");
- } else {
- rs = stmt.executeQuery();
- }
- LOG.debug("Downsampler Query returned ...");
- LOG.info("End Downsampling cycle.");
-
- } catch (SQLException e) {
- LOG.error("Exception during downsampling metrics.", e);
- } finally {
- if (rs != null) {
- try {
- rs.close();
- } catch (SQLException e) {
- // Ignore
- }
- }
- if (stmt != null) {
- try {
- stmt.close();
- } catch (SQLException e) {
- // Ignore
- }
- }
- if (conn != null) {
- try {
- conn.close();
- } catch (SQLException sql) {
- // Ignore
- }
- }
- }
- }
-
- /**
- * Returns the METRIC_NAME NOT LIKE clause if certain metrics or metric patterns are to be skipped
- * since they will be downsampled.
- * @return
- */
- protected String getDownsampledMetricSkipClause() {
-
- //TODO Fix downsampling for UUID change.
- return StringUtils.EMPTY;
-
-// if (CollectionUtils.isEmpty(this.downsampleMetricPatterns)) {
-// return StringUtils.EMPTY;
-// }
-//
-// StringBuilder sb = new StringBuilder();
-//
-// for (int i = 0; i < downsampleMetricPatterns.size(); i++) {
-// sb.append(" METRIC_NAME");
-// sb.append(" NOT");
-// sb.append(" LIKE ");
-// sb.append("'" + downsampleMetricPatterns.get(i) + "'");
-//
-// if (i < downsampleMetricPatterns.size() - 1) {
-// sb.append(" AND ");
-// }
-// }
-//
-// sb.append(" AND ");
-// return sb.toString();
- }
-
- /**
- * Get @AGGREGATOR_TYPE based on the output table.
- * This is solely used by the HAController to determine which lock to acquire.
- */
- public AGGREGATOR_TYPE getAggregatorType() {
- if (outputTableName.contains("RECORD")) {
- return AGGREGATOR_TYPE.HOST;
- } else if (outputTableName.contains("AGGREGATE")) {
- return AGGREGATOR_TYPE.CLUSTER;
- }
- return null;
- }
-
- @Override
- public AGGREGATOR_NAME getName() {
- return aggregatorName;
- }
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/AggregatorUtils.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/AggregatorUtils.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/AggregatorUtils.java
deleted file mode 100644
index b8338fb..0000000
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/AggregatorUtils.java
+++ /dev/null
@@ -1,254 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.aggregators;
-
-
-import java.util.ArrayList;
-import java.util.Date;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.TreeMap;
-
-import org.apache.commons.collections.MapUtils;
-import org.apache.commons.lang.StringUtils;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.metrics2.sink.timeline.PostProcessingUtil;
-import org.apache.hadoop.metrics2.sink.timeline.TimelineMetric;
-
-/**
- *
- */
-public class AggregatorUtils {
-
- private static final Log LOG = LogFactory.getLog(AggregatorUtils.class);
-
- public static double[] calculateAggregates(Map<Long, Double> metricValues) {
- double[] values = new double[4];
- double max = Double.MIN_VALUE;
- double min = Double.MAX_VALUE;
- double sum = 0.0;
- int metricCount = 0;
-
- if (metricValues != null && !metricValues.isEmpty()) {
- for (Double value : metricValues.values()) {
- // TODO: Some nulls in data - need to investigate null values from host
- if (value != null) {
- if (value > max) {
- max = value;
- }
- if (value < min) {
- min = value;
- }
- sum += value;
- }
- }
- metricCount = metricValues.values().size();
- }
- // BR: WHY ZERO is a good idea?
- values[0] = sum;
- values[1] = max != Double.MIN_VALUE ? max : 0.0;
- values[2] = min != Double.MAX_VALUE ? min : 0.0;
- values[3] = metricCount;
-
- return values;
- }
-
- public static Map<TimelineClusterMetric, Double> sliceFromTimelineMetric(
- TimelineMetric timelineMetric, List<Long[]> timeSlices, boolean interpolationEnabled) {
-
- if (timelineMetric.getMetricValues().isEmpty()) {
- return null;
- }
-
- Map<TimelineClusterMetric, Double> timelineClusterMetricMap =
- new HashMap<>();
-
- Long prevTimestamp = -1l;
- TimelineClusterMetric prevMetric = null;
- int count = 0;
- double sum = 0.0;
-
- Map<Long,Double> timeSliceValueMap = new HashMap<>();
- for (Map.Entry<Long, Double> metric : timelineMetric.getMetricValues().entrySet()) {
- if (metric.getValue() == null) {
- continue;
- }
-
- Long timestamp = getSliceTimeForMetric(timeSlices, Long.parseLong(metric.getKey().toString()));
- if (timestamp != -1) {
- // Metric is within desired time range
- TimelineClusterMetric clusterMetric = new TimelineClusterMetric(
- timelineMetric.getMetricName(),
- timelineMetric.getAppId(),
- timelineMetric.getInstanceId(),
- timestamp);
-
- if (prevTimestamp < 0 || timestamp.equals(prevTimestamp)) {
- Double newValue = metric.getValue();
- if (newValue > 0.0) {
- sum += newValue;
- count++;
- }
- } else {
- double metricValue = (count > 0) ? (sum / count) : 0.0;
- timelineClusterMetricMap.put(prevMetric, metricValue);
- timeSliceValueMap.put(prevMetric.getTimestamp(), metricValue);
- sum = metric.getValue();
- count = sum > 0.0 ? 1 : 0;
- }
-
- prevTimestamp = timestamp;
- prevMetric = clusterMetric;
- }
- }
-
- if (prevTimestamp > 0) {
- double metricValue = (count > 0) ? (sum / count) : 0.0;
- timelineClusterMetricMap.put(prevMetric, metricValue);
- timeSliceValueMap.put(prevTimestamp, metricValue);
- }
-
- if (interpolationEnabled) {
- Map<Long, Double> interpolatedValues = interpolateMissingPeriods(timelineMetric.getMetricValues(), timeSlices, timeSliceValueMap, timelineMetric.getType());
- for (Map.Entry<Long, Double> entry : interpolatedValues.entrySet()) {
- TimelineClusterMetric timelineClusterMetric = new TimelineClusterMetric(timelineMetric.getMetricName(), timelineMetric.getAppId(), timelineMetric.getInstanceId(), entry.getKey());
- timelineClusterMetricMap.putIfAbsent(timelineClusterMetric, entry.getValue());
- }
- }
-
- return timelineClusterMetricMap;
- }
-
- private static Map<Long, Double> interpolateMissingPeriods(TreeMap<Long, Double> metricValues,
- List<Long[]> timeSlices,
- Map<Long, Double> timeSliceValueMap, String type) {
- Map<Long, Double> resultClusterMetricMap = new HashMap<>();
-
- if (StringUtils.isNotEmpty(type) && "COUNTER".equalsIgnoreCase(type)) {
- //For Counter Based metrics, ok to do interpolation and extrapolation
-
- List<Long> requiredTimestamps = new ArrayList<>();
- for (Long[] timeSlice : timeSlices) {
- if (!timeSliceValueMap.containsKey(timeSlice[1])) {
- requiredTimestamps.add(timeSlice[1]);
- }
- }
- Map<Long, Double> interpolatedValuesMap = PostProcessingUtil.interpolate(metricValues, requiredTimestamps);
-
- if (interpolatedValuesMap != null) {
- for (Map.Entry<Long, Double> entry : interpolatedValuesMap.entrySet()) {
- Double interpolatedValue = entry.getValue();
-
- if (interpolatedValue != null) {
- resultClusterMetricMap.put( entry.getKey(), interpolatedValue);
- } else {
- LOG.debug("Cannot compute interpolated value, hence skipping.");
- }
- }
- }
- } else {
- //For other metrics, ok to do only interpolation
-
- Double defaultNextSeenValue = null;
- if (MapUtils.isEmpty(timeSliceValueMap) && MapUtils.isNotEmpty(metricValues)) {
- //If no value was found within the start_time based slices, but the metric has value in the server_time range,
- // use that.
-
- Map.Entry<Long,Double> firstEntry = metricValues.firstEntry();
- defaultNextSeenValue = firstEntry.getValue();
- LOG.debug("Found a data point outside timeslice range: " + new Date(firstEntry.getKey()) + ": " + defaultNextSeenValue);
- }
-
- for (int sliceNum = 0; sliceNum < timeSlices.size(); sliceNum++) {
- Long[] timeSlice = timeSlices.get(sliceNum);
-
- if (!timeSliceValueMap.containsKey(timeSlice[1])) {
- LOG.debug("Found an empty slice : " + new Date(timeSlice[0]) + ", " + new Date(timeSlice[1]));
-
- Double lastSeenValue = null;
- int index = sliceNum - 1;
- Long[] prevTimeSlice = null;
- while (lastSeenValue == null && index >= 0) {
- prevTimeSlice = timeSlices.get(index--);
- lastSeenValue = timeSliceValueMap.get(prevTimeSlice[1]);
- }
-
- Double nextSeenValue = null;
- index = sliceNum + 1;
- Long[] nextTimeSlice = null;
- while (nextSeenValue == null && index < timeSlices.size()) {
- nextTimeSlice = timeSlices.get(index++);
- nextSeenValue = timeSliceValueMap.get(nextTimeSlice[1]);
- }
-
- if (nextSeenValue == null) {
- nextSeenValue = defaultNextSeenValue;
- }
-
- Double interpolatedValue = PostProcessingUtil.interpolate(timeSlice[1],
- (prevTimeSlice != null ? prevTimeSlice[1] : null), lastSeenValue,
- (nextTimeSlice != null ? nextTimeSlice[1] : null), nextSeenValue);
-
- if (interpolatedValue != null) {
- LOG.debug("Interpolated value : " + interpolatedValue);
- resultClusterMetricMap.put(timeSlice[1], interpolatedValue);
- } else {
- LOG.debug("Cannot compute interpolated value, hence skipping.");
- }
- }
- }
- }
- return resultClusterMetricMap;
- }
-
- /**
- * Return end of the time slice into which the metric fits.
- */
- public static Long getSliceTimeForMetric(List<Long[]> timeSlices, Long timestamp) {
- for (Long[] timeSlice : timeSlices) {
- if (timestamp >= timeSlice[0] && timestamp < timeSlice[1]) {
- return timeSlice[1];
- }
- }
- return -1l;
- }
-
- /**
- * Return time slices to normalize the timeseries data.
- */
- public static List<Long[]> getTimeSlices(long startTime, long endTime, long timeSliceIntervalMillis) {
- List<Long[]> timeSlices = new ArrayList<Long[]>();
- long sliceStartTime = startTime;
- while (sliceStartTime < endTime) {
- timeSlices.add(new Long[] { sliceStartTime, sliceStartTime + timeSliceIntervalMillis });
- sliceStartTime += timeSliceIntervalMillis;
- }
- return timeSlices;
- }
-
- public static long getRoundedCheckPointTimeMillis(long referenceTime, long aggregatorPeriod) {
- return referenceTime - (referenceTime % aggregatorPeriod);
- }
-
- public static long getRoundedAggregateTimeMillis(long aggregatorPeriod) {
- long currentTime = System.currentTimeMillis();
- return currentTime - (currentTime % aggregatorPeriod);
- }
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/CustomDownSampler.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/CustomDownSampler.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/CustomDownSampler.java
deleted file mode 100644
index cfd82dd..0000000
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/CustomDownSampler.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.aggregators;
-
-import java.util.List;
-
-/**
- * Interface to add a custom downsampler.
- * Each configured downsampler will be executed during an aggregation cycle.
- */
-public interface CustomDownSampler {
-
- /**
- * Gatekeeper to check the configs. If this fails, the downsampling is not done.
- * @return
- */
- public boolean validateConfigs();
-
- /**
- * Return the set of statements that needs to be executed for the downsampling.
- * @param startTime
- * @param endTime
- * @param tableName
- * @return
- */
- public List<String> prepareDownSamplingStatement(Long startTime, Long endTime, String tableName);
-
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/DownSamplerUtils.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/DownSamplerUtils.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/DownSamplerUtils.java
deleted file mode 100644
index 23b1cb3..0000000
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/DownSamplerUtils.java
+++ /dev/null
@@ -1,117 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.aggregators;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-import org.apache.commons.lang.StringUtils;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-
-/**
- * DownSampler utility class. Responsible for fetching downsampler configs from Metrics config, and determine if
- * any downsamplers are configured.
- */
-
-public class DownSamplerUtils {
-
- public static final String downSamplerConfigPrefix = "timeline.metrics.downsampler.";
- public static final String downSamplerMetricPatternsConfig = "metric.patterns";
- public static final String topNDownSampler = "topn";
- private static final Log LOG = LogFactory.getLog(DownSamplerUtils.class);
-
-
-
- /**
- * Get the list of metrics that are requested to be downsampled.
- * @param configuration
- * @return List of metric patterns/names that are to be downsampled.
- */
- public static List<String> getDownsampleMetricPatterns(Configuration configuration) {
- Map<String, String> conf = configuration.getValByRegex(downSamplerConfigPrefix + "*");
- List<String> metricPatterns = new ArrayList<>();
- Set<String> keys = conf.keySet();
- for (String key : keys) {
- if (key.endsWith(downSamplerMetricPatternsConfig)) {
- String patternString = conf.get(key);
- String[] patterns = StringUtils.split(patternString, ",");
- for (String pattern : patterns) {
- if (StringUtils.isNotEmpty(pattern)) {
- String trimmedPattern = pattern.trim();
- metricPatterns.add(trimmedPattern);
- }
- }
- }
- }
- return metricPatterns;
- }
-
- /**
- * Get the list of downsamplers that are configured in ams-site
- * Sample config
- <name>timeline.metrics.downsampler.topn.metric.patterns</name>
- <value>dfs.NNTopUserOpCounts.windowMs=60000.op%,dfs.NNTopUserOpCounts.windowMs=300000.op%</value>
-
- <name>timeline.metrics.downsampler.topn.value</name>
- <value>10</value>
-
- <name>timeline.metrics.downsampler.topn.function</name>
- <value>max</value>
- * @param configuration
- * @return
- */
- public static List<CustomDownSampler> getDownSamplers(Configuration configuration) {
-
- Map<String,String> conf = configuration.getValByRegex(downSamplerConfigPrefix + "*");
- List<CustomDownSampler> downSamplers = new ArrayList<>();
- Set<String> keys = conf.keySet();
-
- try {
- for (String key : keys) {
- if (key.startsWith(downSamplerConfigPrefix) && key.endsWith(downSamplerMetricPatternsConfig)) {
- String type = key.split("\\.")[3];
- CustomDownSampler downSampler = getDownSamplerByType(type, conf);
- if (downSampler != null) {
- downSamplers.add(downSampler);
- }
- }
- }
- } catch (Exception e) {
- LOG.warn("Exception caught while parsing downsampler configs from ams-site : " + e.getMessage());
- }
- return downSamplers;
- }
-
- public static CustomDownSampler getDownSamplerByType(String type, Map<String, String> conf) {
- if (type == null) {
- return null;
- }
-
- if (StringUtils.isNotEmpty(type) && type.equalsIgnoreCase(topNDownSampler)) {
- return TopNDownSampler.fromConfig(conf);
- }
-
- LOG.warn("Unknown downsampler requested : " + type);
- return null;
- }
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/Function.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/Function.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/Function.java
deleted file mode 100644
index ab9d2e9..0000000
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/Function.java
+++ /dev/null
@@ -1,212 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.aggregators;
-
-import java.util.Arrays;
-
-/**
- * Is used to determine metrics aggregate table.
- *
- * @see org.apache.hadoop.yarn.server.applicationhistoryservice.webapp.TimelineWebServices#getTimelineMetric
- * @see org.apache.hadoop.yarn.server.applicationhistoryservice.webapp.TimelineWebServices#getTimelineMetrics
- */
-public class Function {
- public static Function DEFAULT_VALUE_FUNCTION = new Function(ReadFunction.VALUE, null);
- private static final String SUFFIX_SEPARATOR = "\\._";
-
- private ReadFunction readFunction = ReadFunction.VALUE;
- private PostProcessingFunction postProcessingFunction = null;
-
- public Function() {
- }
-
- public Function(ReadFunction readFunction,
- PostProcessingFunction ppFunction){
- if (readFunction!=null){
- this.readFunction = readFunction ;
- }
- this.postProcessingFunction = ppFunction;
- }
-
- /**
- * Segregate post processing function eg: rate from aggregate function,
- * example: avg, in any order
- * @param metricName metric name from request
- * @return @Function
- */
- public static Function fromMetricName(String metricName) {
- // gets postprocessing, and aggregation function
- // ex. Metric._rate._avg
- String[] parts = metricName.split(SUFFIX_SEPARATOR);
-
- ReadFunction readFunction = ReadFunction.VALUE;
- PostProcessingFunction ppFunction = null;
-
- if (parts.length <= 1) {
- return new Function(readFunction, null);
- }
- if (parts.length > 3) {
- throw new IllegalArgumentException("Invalid number of functions specified.");
- }
-
- // Parse functions
- boolean isSuccessful = false; // Best effort
- for (String part : parts) {
- if (ReadFunction.isPresent(part)) {
- readFunction = ReadFunction.getFunction(part);
- isSuccessful = true;
- }
- if (PostProcessingFunction.isPresent(part)) {
- ppFunction = PostProcessingFunction.getFunction(part);
- isSuccessful = true;
- }
- }
-
- // Throw exception if parsing failed
- if (!isSuccessful) {
- throw new FunctionFormatException("Could not parse provided functions: " +
- "" + Arrays.asList(parts));
- }
-
- return new Function(readFunction, ppFunction);
- }
-
- public String getSuffix(){
- return (postProcessingFunction == null)? readFunction.getSuffix() :
- postProcessingFunction.getSuffix() + readFunction.getSuffix();
- }
-
- public ReadFunction getReadFunction() {
- return readFunction;
- }
-
- @Override
- public String toString() {
- return "Function{" +
- "readFunction=" + readFunction +
- ", postProcessingFunction=" + postProcessingFunction +
- '}';
- }
-
- @Override
- public boolean equals(Object o) {
- if (this == o) return true;
- if (!(o instanceof Function)) return false;
-
- Function function = (Function) o;
-
- return postProcessingFunction == function.postProcessingFunction
- && readFunction == function.readFunction;
-
- }
-
- @Override
- public int hashCode() {
- int result = readFunction.hashCode();
- result = 31 * result + (postProcessingFunction != null ?
- postProcessingFunction.hashCode() : 0);
- return result;
- }
-
- public enum PostProcessingFunction {
- NONE(""),
- RATE("._rate"),
- DIFF("._diff");
-
- PostProcessingFunction(String suffix){
- this.suffix = suffix;
- }
-
- private String suffix = "";
-
- public String getSuffix(){
- return suffix;
- }
-
- public static boolean isPresent(String functionName) {
- try {
- PostProcessingFunction.valueOf(functionName.toUpperCase());
- } catch (IllegalArgumentException e) {
- return false;
- }
- return true;
- }
-
- public static PostProcessingFunction getFunction(String functionName) throws FunctionFormatException {
- if (functionName == null) {
- return NONE;
- }
-
- try {
- return PostProcessingFunction.valueOf(functionName.toUpperCase());
- } catch (IllegalArgumentException e) {
- throw new FunctionFormatException("Function should be ._rate", e);
- }
- }
- }
-
- public enum ReadFunction {
- VALUE(""),
- AVG("._avg"),
- MIN("._min"),
- MAX("._max"),
- SUM("._sum");
-
- private final String suffix;
-
- ReadFunction(String suffix){
- this.suffix = suffix;
- }
-
- public String getSuffix() {
- return suffix;
- }
-
- public static boolean isPresent(String functionName) {
- try {
- ReadFunction.valueOf(functionName.toUpperCase());
- } catch (IllegalArgumentException e) {
- return false;
- }
- return true;
- }
-
- public static ReadFunction getFunction(String functionName) throws FunctionFormatException {
- if (functionName == null) {
- return VALUE;
- }
- try {
- return ReadFunction.valueOf(functionName.toUpperCase());
- } catch (IllegalArgumentException e) {
- throw new FunctionFormatException(
- "Function should be sum, avg, min, max. Got " + functionName, e);
- }
- }
- }
-
- public static class FunctionFormatException extends IllegalArgumentException {
- public FunctionFormatException(String message) {
- super(message);
- }
-
- public FunctionFormatException(String message, Throwable cause) {
- super(message, cause);
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineClusterMetric.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineClusterMetric.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineClusterMetric.java
deleted file mode 100644
index 6e793e1..0000000
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineClusterMetric.java
+++ /dev/null
@@ -1,97 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.aggregators;
-
-public class TimelineClusterMetric {
- private String metricName;
- private String appId;
- private String instanceId;
- private long timestamp;
-
- public TimelineClusterMetric(String metricName, String appId, String instanceId,
- long timestamp) {
- this.metricName = metricName;
- this.appId = appId;
- this.instanceId = instanceId;
- this.timestamp = timestamp;
- }
-
- public String getMetricName() {
- return metricName;
- }
-
- public String getAppId() {
- return appId;
- }
-
- public String getInstanceId() {
- return instanceId;
- }
-
- public long getTimestamp() {
- return timestamp;
- }
-
- @Override
- public boolean equals(Object o) {
- if (this == o) return true;
- if (o == null || getClass() != o.getClass()) return false;
-
- TimelineClusterMetric that = (TimelineClusterMetric) o;
-
- if (timestamp != that.timestamp) return false;
- if (appId != null ? !appId.equals(that.appId) : that.appId != null)
- return false;
- if (instanceId != null ? !instanceId.equals(that.instanceId) : that.instanceId != null)
- return false;
- if (!metricName.equals(that.metricName)) return false;
-
- return true;
- }
-
- public boolean equalsExceptTime(TimelineClusterMetric metric) {
- if (!metricName.equals(metric.metricName)) return false;
- if (!appId.equals(metric.appId)) return false;
- if (instanceId != null ? !instanceId.equals(metric.instanceId) : metric.instanceId != null)
- return false;
-
- return true;
- }
- @Override
- public int hashCode() {
- int result = metricName.hashCode();
- result = 31 * result + (appId != null ? appId.hashCode() : 0);
- result = 31 * result + (instanceId != null ? instanceId.hashCode() : 0);
- result = 31 * result + (int) (timestamp ^ (timestamp >>> 32));
- return result;
- }
-
- @Override
- public String toString() {
- return "TimelineClusterMetric{" +
- "metricName='" + metricName + '\'' +
- ", appId='" + appId + '\'' +
- ", instanceId='" + instanceId + '\'' +
- ", timestamp=" + timestamp +
- '}';
- }
-
- public void setTimestamp(long timestamp) {
- this.timestamp = timestamp;
- }
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricAggregator.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricAggregator.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricAggregator.java
deleted file mode 100644
index 150e3f1..0000000
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricAggregator.java
+++ /dev/null
@@ -1,59 +0,0 @@
-package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.aggregators;
-
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.availability.AggregationTaskRunner.AGGREGATOR_NAME;
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p/>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p/>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-public interface TimelineMetricAggregator extends Runnable {
- /**
- * Aggregate metric data within the time bounds.
- *
- * @param startTime start time millis
- * @param endTime end time millis
- * @return success
- */
- boolean doWork(long startTime, long endTime);
-
- /**
- * Is aggregator is disabled by configuration.
- *
- * @return true/false
- */
- boolean isDisabled();
-
- /**
- * Return aggregator Interval
- *
- * @return Interval in Millis
- */
- Long getSleepIntervalMillis();
-
- /**
- * Get aggregator name
- * @return @AGGREGATOR_NAME
- */
- AGGREGATOR_NAME getName();
-
- /**
- * Known aggregator types
- */
- enum AGGREGATOR_TYPE {
- CLUSTER,
- HOST
- }
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricAggregatorFactory.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricAggregatorFactory.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricAggregatorFactory.java
deleted file mode 100644
index 3728d19..0000000
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricAggregatorFactory.java
+++ /dev/null
@@ -1,528 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.aggregators;
-
-import static java.util.concurrent.TimeUnit.SECONDS;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.CLUSTER_AGGREGATOR_DAILY_CHECKPOINT_CUTOFF_MULTIPLIER;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.CLUSTER_AGGREGATOR_DAILY_DISABLED;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.CLUSTER_AGGREGATOR_DAILY_SLEEP_INTERVAL;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.CLUSTER_AGGREGATOR_HOUR_CHECKPOINT_CUTOFF_MULTIPLIER;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.CLUSTER_AGGREGATOR_HOUR_DISABLED;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.CLUSTER_AGGREGATOR_HOUR_SLEEP_INTERVAL;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.CLUSTER_AGGREGATOR_MINUTE_CHECKPOINT_CUTOFF_MULTIPLIER;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.CLUSTER_AGGREGATOR_MINUTE_DISABLED;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.CLUSTER_AGGREGATOR_MINUTE_SLEEP_INTERVAL;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.CLUSTER_AGGREGATOR_SECOND_CHECKPOINT_CUTOFF_MULTIPLIER;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.CLUSTER_AGGREGATOR_SECOND_DISABLED;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.CLUSTER_AGGREGATOR_SECOND_SLEEP_INTERVAL;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.CLUSTER_AGGREGATOR_TIMESLICE_INTERVAL;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.DEFAULT_CHECKPOINT_LOCATION;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.HOST_AGGREGATOR_DAILY_CHECKPOINT_CUTOFF_MULTIPLIER;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.HOST_AGGREGATOR_DAILY_DISABLED;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.HOST_AGGREGATOR_DAILY_SLEEP_INTERVAL;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.HOST_AGGREGATOR_HOUR_CHECKPOINT_CUTOFF_MULTIPLIER;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.HOST_AGGREGATOR_HOUR_DISABLED;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.HOST_AGGREGATOR_HOUR_SLEEP_INTERVAL;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.HOST_AGGREGATOR_MINUTE_CHECKPOINT_CUTOFF_MULTIPLIER;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.HOST_AGGREGATOR_MINUTE_DISABLED;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.HOST_AGGREGATOR_MINUTE_SLEEP_INTERVAL;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.TIMELINE_METRICS_AGGREGATOR_CHECKPOINT_DIR;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.USE_GROUPBY_AGGREGATOR_QUERIES;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.availability.AggregationTaskRunner.AGGREGATOR_NAME.METRIC_AGGREGATE_DAILY;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.availability.AggregationTaskRunner.AGGREGATOR_NAME.METRIC_AGGREGATE_HOURLY;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.availability.AggregationTaskRunner.AGGREGATOR_NAME.METRIC_AGGREGATE_MINUTE;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.availability.AggregationTaskRunner.AGGREGATOR_NAME.METRIC_AGGREGATE_SECOND;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.availability.AggregationTaskRunner.AGGREGATOR_NAME.METRIC_RECORD_DAILY;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.availability.AggregationTaskRunner.AGGREGATOR_NAME.METRIC_RECORD_HOURLY;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.availability.AggregationTaskRunner.AGGREGATOR_NAME.METRIC_RECORD_MINUTE;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query.PhoenixTransactSQL.METRICS_AGGREGATE_DAILY_TABLE_NAME;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query.PhoenixTransactSQL.METRICS_AGGREGATE_HOURLY_TABLE_NAME;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query.PhoenixTransactSQL.METRICS_AGGREGATE_MINUTE_TABLE_NAME;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query.PhoenixTransactSQL.METRICS_CLUSTER_AGGREGATE_DAILY_TABLE_NAME;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query.PhoenixTransactSQL.METRICS_CLUSTER_AGGREGATE_HOURLY_TABLE_NAME;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query.PhoenixTransactSQL.METRICS_CLUSTER_AGGREGATE_MINUTE_TABLE_NAME;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query.PhoenixTransactSQL.METRICS_CLUSTER_AGGREGATE_TABLE_NAME;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query.PhoenixTransactSQL.METRICS_RECORD_TABLE_NAME;
-
-import java.util.concurrent.ConcurrentHashMap;
-
-import org.apache.commons.io.FilenameUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.PhoenixHBaseAccessor;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricDistributedCache;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.availability.MetricCollectorHAController;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.discovery.TimelineMetricMetadataManager;
-
-/**
- * Factory class that knows how to create a aggregator instance using
- * TimelineMetricConfiguration
- */
-public class TimelineMetricAggregatorFactory {
- private static final String HOST_AGGREGATE_MINUTE_CHECKPOINT_FILE =
- "timeline-metrics-host-aggregator-checkpoint";
- private static final String HOST_AGGREGATE_HOURLY_CHECKPOINT_FILE =
- "timeline-metrics-host-aggregator-hourly-checkpoint";
- private static final String HOST_AGGREGATE_DAILY_CHECKPOINT_FILE =
- "timeline-metrics-host-aggregator-daily-checkpoint";
- private static final String CLUSTER_AGGREGATOR_CHECKPOINT_FILE =
- "timeline-metrics-cluster-aggregator-checkpoint";
- private static final String CLUSTER_AGGREGATOR_MINUTE_CHECKPOINT_FILE =
- "timeline-metrics-cluster-aggregator-minute-checkpoint";
- private static final String CLUSTER_AGGREGATOR_HOURLY_CHECKPOINT_FILE =
- "timeline-metrics-cluster-aggregator-hourly-checkpoint";
- private static final String CLUSTER_AGGREGATOR_DAILY_CHECKPOINT_FILE =
- "timeline-metrics-cluster-aggregator-daily-checkpoint";
-
- private static boolean useGroupByAggregator(Configuration metricsConf) {
- return Boolean.parseBoolean(metricsConf.get(USE_GROUPBY_AGGREGATOR_QUERIES, "true"));
- }
-
- /**
- * Minute based aggregation for hosts.
- * Interval : 5 mins
- */
- public static TimelineMetricAggregator createTimelineMetricAggregatorMinute
- (PhoenixHBaseAccessor hBaseAccessor, Configuration metricsConf,
- TimelineMetricMetadataManager metadataManager,
- MetricCollectorHAController haController) {
-
- String checkpointDir = metricsConf.get(
- TIMELINE_METRICS_AGGREGATOR_CHECKPOINT_DIR, DEFAULT_CHECKPOINT_LOCATION);
- String checkpointLocation = FilenameUtils.concat(checkpointDir,
- HOST_AGGREGATE_MINUTE_CHECKPOINT_FILE);
- long sleepIntervalMillis = SECONDS.toMillis(metricsConf.getLong
- (HOST_AGGREGATOR_MINUTE_SLEEP_INTERVAL, 300l)); // 5 mins
-
- int checkpointCutOffMultiplier = metricsConf.getInt
- (HOST_AGGREGATOR_MINUTE_CHECKPOINT_CUTOFF_MULTIPLIER, 3);
- String hostAggregatorDisabledParam = HOST_AGGREGATOR_MINUTE_DISABLED;
-
- String inputTableName = METRICS_RECORD_TABLE_NAME;
- String outputTableName = METRICS_AGGREGATE_MINUTE_TABLE_NAME;
-
- if (useGroupByAggregator(metricsConf)) {
- return new org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.aggregators.v2.TimelineMetricHostAggregator(
- METRIC_RECORD_MINUTE,
- hBaseAccessor, metricsConf,
- checkpointLocation,
- sleepIntervalMillis,
- checkpointCutOffMultiplier,
- hostAggregatorDisabledParam,
- inputTableName,
- outputTableName,
- 120000l,
- haController
- );
- }
-
- return new TimelineMetricHostAggregator(
- METRIC_RECORD_MINUTE,
- metadataManager,
- hBaseAccessor, metricsConf,
- checkpointLocation,
- sleepIntervalMillis,
- checkpointCutOffMultiplier,
- hostAggregatorDisabledParam,
- inputTableName,
- outputTableName,
- 120000l,
- haController);
- }
-
- /**
- * Hourly aggregation for hosts.
- * Interval : 1 hour
- */
- public static TimelineMetricAggregator createTimelineMetricAggregatorHourly
- (PhoenixHBaseAccessor hBaseAccessor, Configuration metricsConf,
- TimelineMetricMetadataManager metadataManager,
- MetricCollectorHAController haController) {
-
- String checkpointDir = metricsConf.get(
- TIMELINE_METRICS_AGGREGATOR_CHECKPOINT_DIR, DEFAULT_CHECKPOINT_LOCATION);
- String checkpointLocation = FilenameUtils.concat(checkpointDir,
- HOST_AGGREGATE_HOURLY_CHECKPOINT_FILE);
- long sleepIntervalMillis = SECONDS.toMillis(metricsConf.getLong
- (HOST_AGGREGATOR_HOUR_SLEEP_INTERVAL, 3600l));
-
- int checkpointCutOffMultiplier = metricsConf.getInt
- (HOST_AGGREGATOR_HOUR_CHECKPOINT_CUTOFF_MULTIPLIER, 2);
- String hostAggregatorDisabledParam = HOST_AGGREGATOR_HOUR_DISABLED;
-
- String inputTableName = METRICS_AGGREGATE_MINUTE_TABLE_NAME;
- String outputTableName = METRICS_AGGREGATE_HOURLY_TABLE_NAME;
-
- if (useGroupByAggregator(metricsConf)) {
- return new org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.aggregators.v2.TimelineMetricHostAggregator(
- METRIC_RECORD_HOURLY,
- hBaseAccessor, metricsConf,
- checkpointLocation,
- sleepIntervalMillis,
- checkpointCutOffMultiplier,
- hostAggregatorDisabledParam,
- inputTableName,
- outputTableName,
- 3600000l,
- haController
- );
- }
-
- return new TimelineMetricHostAggregator(
- METRIC_RECORD_HOURLY,
- metadataManager,
- hBaseAccessor, metricsConf,
- checkpointLocation,
- sleepIntervalMillis,
- checkpointCutOffMultiplier,
- hostAggregatorDisabledParam,
- inputTableName,
- outputTableName,
- 3600000l,
- haController);
- }
-
- /**
- * Daily aggregation for hosts.
- * Interval : 1 day
- */
- public static TimelineMetricAggregator createTimelineMetricAggregatorDaily
- (PhoenixHBaseAccessor hBaseAccessor, Configuration metricsConf,
- TimelineMetricMetadataManager metadataManager,
- MetricCollectorHAController haController) {
-
- String checkpointDir = metricsConf.get(
- TIMELINE_METRICS_AGGREGATOR_CHECKPOINT_DIR, DEFAULT_CHECKPOINT_LOCATION);
- String checkpointLocation = FilenameUtils.concat(checkpointDir,
- HOST_AGGREGATE_DAILY_CHECKPOINT_FILE);
- long sleepIntervalMillis = SECONDS.toMillis(metricsConf.getLong
- (HOST_AGGREGATOR_DAILY_SLEEP_INTERVAL, 86400l));
-
- int checkpointCutOffMultiplier = metricsConf.getInt
- (HOST_AGGREGATOR_DAILY_CHECKPOINT_CUTOFF_MULTIPLIER, 1);
- String hostAggregatorDisabledParam = HOST_AGGREGATOR_DAILY_DISABLED;
-
- String inputTableName = METRICS_AGGREGATE_HOURLY_TABLE_NAME;
- String outputTableName = METRICS_AGGREGATE_DAILY_TABLE_NAME;
-
- if (useGroupByAggregator(metricsConf)) {
- return new org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.aggregators.v2.TimelineMetricHostAggregator(
- METRIC_RECORD_DAILY,
- hBaseAccessor, metricsConf,
- checkpointLocation,
- sleepIntervalMillis,
- checkpointCutOffMultiplier,
- hostAggregatorDisabledParam,
- inputTableName,
- outputTableName,
- 3600000l,
- haController
- );
- }
-
- return new TimelineMetricHostAggregator(
- METRIC_RECORD_DAILY,
- metadataManager,
- hBaseAccessor, metricsConf,
- checkpointLocation,
- sleepIntervalMillis,
- checkpointCutOffMultiplier,
- hostAggregatorDisabledParam,
- inputTableName,
- outputTableName,
- 3600000l,
- haController);
- }
-
- /**
- * Second aggregation for cluster.
- * Interval : 2 mins
- * Timeslice : 30 sec
- */
- public static TimelineMetricAggregator createTimelineClusterAggregatorSecond(
- PhoenixHBaseAccessor hBaseAccessor, Configuration metricsConf,
- TimelineMetricMetadataManager metadataManager,
- MetricCollectorHAController haController,
- TimelineMetricDistributedCache distributedCache) {
-
- String checkpointDir = metricsConf.get(
- TIMELINE_METRICS_AGGREGATOR_CHECKPOINT_DIR, DEFAULT_CHECKPOINT_LOCATION);
-
- String checkpointLocation = FilenameUtils.concat(checkpointDir,
- CLUSTER_AGGREGATOR_CHECKPOINT_FILE);
-
- long sleepIntervalMillis = SECONDS.toMillis(metricsConf.getLong
- (CLUSTER_AGGREGATOR_SECOND_SLEEP_INTERVAL, 120l));
-
- long timeSliceIntervalMillis = SECONDS.toMillis(metricsConf.getInt
- (CLUSTER_AGGREGATOR_TIMESLICE_INTERVAL, 30));
-
- int checkpointCutOffMultiplier =
- metricsConf.getInt(CLUSTER_AGGREGATOR_SECOND_CHECKPOINT_CUTOFF_MULTIPLIER, 2);
-
- String outputTableName = METRICS_CLUSTER_AGGREGATE_TABLE_NAME;
- String aggregatorDisabledParam = CLUSTER_AGGREGATOR_SECOND_DISABLED;
-
- // Second based aggregation have added responsibility of time slicing
- if (TimelineMetricConfiguration.getInstance().isCollectorInMemoryAggregationEnabled()) {
- return new TimelineMetricClusterAggregatorSecondWithCacheSource(
- METRIC_AGGREGATE_SECOND,
- metadataManager,
- hBaseAccessor, metricsConf,
- checkpointLocation,
- sleepIntervalMillis,
- checkpointCutOffMultiplier,
- aggregatorDisabledParam,
- null,
- outputTableName,
- 120000l,
- timeSliceIntervalMillis,
- haController,
- distributedCache
- );
- }
-
- String inputTableName = METRICS_RECORD_TABLE_NAME;
- return new TimelineMetricClusterAggregatorSecond(
- METRIC_AGGREGATE_SECOND,
- metadataManager,
- hBaseAccessor, metricsConf,
- checkpointLocation,
- sleepIntervalMillis,
- checkpointCutOffMultiplier,
- aggregatorDisabledParam,
- inputTableName,
- outputTableName,
- 120000l,
- timeSliceIntervalMillis,
- haController
- );
- }
-
- /**
- * Minute aggregation for cluster.
- * Interval : 5 mins
- */
- public static TimelineMetricAggregator createTimelineClusterAggregatorMinute(
- PhoenixHBaseAccessor hBaseAccessor, Configuration metricsConf,
- TimelineMetricMetadataManager metadataManager,
- MetricCollectorHAController haController) {
-
- String checkpointDir = metricsConf.get(
- TIMELINE_METRICS_AGGREGATOR_CHECKPOINT_DIR, DEFAULT_CHECKPOINT_LOCATION);
-
- String checkpointLocation = FilenameUtils.concat(checkpointDir,
- CLUSTER_AGGREGATOR_MINUTE_CHECKPOINT_FILE);
-
- long sleepIntervalMillis = SECONDS.toMillis(metricsConf.getLong
- (CLUSTER_AGGREGATOR_MINUTE_SLEEP_INTERVAL, 300l));
-
- int checkpointCutOffMultiplier = metricsConf.getInt
- (CLUSTER_AGGREGATOR_MINUTE_CHECKPOINT_CUTOFF_MULTIPLIER, 2);
-
- String inputTableName = METRICS_CLUSTER_AGGREGATE_TABLE_NAME;
- String outputTableName = METRICS_CLUSTER_AGGREGATE_MINUTE_TABLE_NAME;
- String aggregatorDisabledParam = CLUSTER_AGGREGATOR_MINUTE_DISABLED;
-
- if (useGroupByAggregator(metricsConf)) {
- return new org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.aggregators.v2.TimelineMetricClusterAggregator(
- METRIC_AGGREGATE_MINUTE,
- hBaseAccessor, metricsConf,
- checkpointLocation,
- sleepIntervalMillis,
- checkpointCutOffMultiplier,
- aggregatorDisabledParam,
- inputTableName,
- outputTableName,
- 120000l,
- haController
- );
- }
-
- return new TimelineMetricClusterAggregator(
- METRIC_AGGREGATE_MINUTE,
- metadataManager,
- hBaseAccessor, metricsConf,
- checkpointLocation,
- sleepIntervalMillis,
- checkpointCutOffMultiplier,
- aggregatorDisabledParam,
- inputTableName,
- outputTableName,
- 120000l,
- haController
- );
- }
-
- /**
- * Hourly aggregation for cluster.
- * Interval : 1 hour
- */
- public static TimelineMetricAggregator createTimelineClusterAggregatorHourly(
- PhoenixHBaseAccessor hBaseAccessor, Configuration metricsConf,
- TimelineMetricMetadataManager metadataManager,
- MetricCollectorHAController haController) {
-
- String checkpointDir = metricsConf.get(
- TIMELINE_METRICS_AGGREGATOR_CHECKPOINT_DIR, DEFAULT_CHECKPOINT_LOCATION);
-
- String checkpointLocation = FilenameUtils.concat(checkpointDir,
- CLUSTER_AGGREGATOR_HOURLY_CHECKPOINT_FILE);
-
- long sleepIntervalMillis = SECONDS.toMillis(metricsConf.getLong
- (CLUSTER_AGGREGATOR_HOUR_SLEEP_INTERVAL, 3600l));
-
- int checkpointCutOffMultiplier = metricsConf.getInt
- (CLUSTER_AGGREGATOR_HOUR_CHECKPOINT_CUTOFF_MULTIPLIER, 2);
-
- String inputTableName = METRICS_CLUSTER_AGGREGATE_TABLE_NAME;
- String outputTableName = METRICS_CLUSTER_AGGREGATE_HOURLY_TABLE_NAME;
- String aggregatorDisabledParam = CLUSTER_AGGREGATOR_HOUR_DISABLED;
-
- if (useGroupByAggregator(metricsConf)) {
- return new org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.aggregators.v2.TimelineMetricClusterAggregator(
- METRIC_AGGREGATE_HOURLY,
- hBaseAccessor, metricsConf,
- checkpointLocation,
- sleepIntervalMillis,
- checkpointCutOffMultiplier,
- aggregatorDisabledParam,
- inputTableName,
- outputTableName,
- 120000l,
- haController
- );
- }
-
- return new TimelineMetricClusterAggregator(
- METRIC_AGGREGATE_HOURLY,
- metadataManager,
- hBaseAccessor, metricsConf,
- checkpointLocation,
- sleepIntervalMillis,
- checkpointCutOffMultiplier,
- aggregatorDisabledParam,
- inputTableName,
- outputTableName,
- 120000l,
- haController
- );
- }
-
- /**
- * Daily aggregation for cluster.
- * Interval : 1 day
- */
- public static TimelineMetricAggregator createTimelineClusterAggregatorDaily(
- PhoenixHBaseAccessor hBaseAccessor, Configuration metricsConf,
- TimelineMetricMetadataManager metadataManager,
- MetricCollectorHAController haController) {
-
- String checkpointDir = metricsConf.get(
- TIMELINE_METRICS_AGGREGATOR_CHECKPOINT_DIR, DEFAULT_CHECKPOINT_LOCATION);
-
- String checkpointLocation = FilenameUtils.concat(checkpointDir,
- CLUSTER_AGGREGATOR_DAILY_CHECKPOINT_FILE);
-
- long sleepIntervalMillis = SECONDS.toMillis(metricsConf.getLong
- (CLUSTER_AGGREGATOR_DAILY_SLEEP_INTERVAL, 86400l));
-
- int checkpointCutOffMultiplier = metricsConf.getInt
- (CLUSTER_AGGREGATOR_DAILY_CHECKPOINT_CUTOFF_MULTIPLIER, 1);
-
- String inputTableName = METRICS_CLUSTER_AGGREGATE_HOURLY_TABLE_NAME;
- String outputTableName = METRICS_CLUSTER_AGGREGATE_DAILY_TABLE_NAME;
- String aggregatorDisabledParam = CLUSTER_AGGREGATOR_DAILY_DISABLED;
-
- if (useGroupByAggregator(metricsConf)) {
- return new org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.aggregators.v2.TimelineMetricClusterAggregator(
- METRIC_AGGREGATE_DAILY,
- hBaseAccessor, metricsConf,
- checkpointLocation,
- sleepIntervalMillis,
- checkpointCutOffMultiplier,
- aggregatorDisabledParam,
- inputTableName,
- outputTableName,
- 120000l,
- haController
- );
- }
-
- return new TimelineMetricClusterAggregator(
- METRIC_AGGREGATE_DAILY,
- metadataManager,
- hBaseAccessor, metricsConf,
- checkpointLocation,
- sleepIntervalMillis,
- checkpointCutOffMultiplier,
- aggregatorDisabledParam,
- inputTableName,
- outputTableName,
- 120000l,
- haController
- );
- }
-
- public static TimelineMetricAggregator createFilteringTimelineMetricAggregatorMinute(PhoenixHBaseAccessor hBaseAccessor, Configuration metricsConf, TimelineMetricMetadataManager metricMetadataManager, MetricCollectorHAController haController, ConcurrentHashMap<String, Long> postedAggregatedMap) {
- String checkpointDir = metricsConf.get(
- TIMELINE_METRICS_AGGREGATOR_CHECKPOINT_DIR, DEFAULT_CHECKPOINT_LOCATION);
- String checkpointLocation = FilenameUtils.concat(checkpointDir,
- HOST_AGGREGATE_MINUTE_CHECKPOINT_FILE);
- long sleepIntervalMillis = SECONDS.toMillis(metricsConf.getLong
- (HOST_AGGREGATOR_MINUTE_SLEEP_INTERVAL, 300l)); // 5 mins
-
- int checkpointCutOffMultiplier = metricsConf.getInt
- (HOST_AGGREGATOR_MINUTE_CHECKPOINT_CUTOFF_MULTIPLIER, 3);
- String hostAggregatorDisabledParam = HOST_AGGREGATOR_MINUTE_DISABLED;
-
- String inputTableName = METRICS_RECORD_TABLE_NAME;
- String outputTableName = METRICS_AGGREGATE_MINUTE_TABLE_NAME;
-
- if (useGroupByAggregator(metricsConf)) {
- return new org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.aggregators.v2.TimelineMetricFilteringHostAggregator(
- METRIC_RECORD_MINUTE,
- metricMetadataManager,
- hBaseAccessor, metricsConf,
- checkpointLocation,
- sleepIntervalMillis,
- checkpointCutOffMultiplier,
- hostAggregatorDisabledParam,
- inputTableName,
- outputTableName,
- 120000l,
- haController,
- postedAggregatedMap
- );
- }
-
- return new TimelineMetricFilteringHostAggregator(
- METRIC_RECORD_MINUTE,
- metricMetadataManager,
- hBaseAccessor, metricsConf,
- checkpointLocation,
- sleepIntervalMillis,
- checkpointCutOffMultiplier,
- hostAggregatorDisabledParam,
- inputTableName,
- outputTableName,
- 120000l,
- haController,
- postedAggregatedMap);
- }
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricAppAggregator.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricAppAggregator.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricAppAggregator.java
deleted file mode 100644
index b06b147..0000000
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricAppAggregator.java
+++ /dev/null
@@ -1,189 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.aggregators;
-
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.CLUSTER_AGGREGATOR_APP_IDS;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.HOST_APP_ID;
-
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
-
-import org.apache.commons.lang.StringUtils;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.metrics2.sink.timeline.MetricClusterAggregate;
-import org.apache.hadoop.metrics2.sink.timeline.TimelineMetricMetadata;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricsFilter;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.discovery.TimelineMetricHostMetadata;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.discovery.TimelineMetricMetadataKey;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.discovery.TimelineMetricMetadataManager;
-
-/**
- * Aggregator responsible for providing app level host aggregates. This task
- * is accomplished without doing a round trip to storage, rather
- * TimelineMetricClusterAggregators are responsible for lifecycle of
- * @TimelineMetricAppAggregator and provide the raw data to aggregate.
- */
-public class TimelineMetricAppAggregator {
- private static final Log LOG = LogFactory.getLog(TimelineMetricAppAggregator.class);
- // Lookup to check candidacy of an app
- private final List<String> appIdsToAggregate;
- private final Map<String, TimelineMetricHostMetadata> hostMetadata;
- Map<TimelineClusterMetric, MetricClusterAggregate> aggregateClusterMetrics = new HashMap<>();
- TimelineMetricMetadataManager metadataManagerInstance;
-
- public TimelineMetricAppAggregator(TimelineMetricMetadataManager metadataManager,
- Configuration metricsConf) {
- appIdsToAggregate = getAppIdsForHostAggregation(metricsConf);
- hostMetadata = metadataManager.getHostedAppsCache();
- metadataManagerInstance = metadataManager;
- LOG.info("AppIds configured for aggregation: " + appIdsToAggregate);
- }
-
- /**
- * Lifecycle method to initialize aggregation cycle.
- */
- public void init() {
- LOG.debug("Initializing aggregation cycle.");
- aggregateClusterMetrics = new HashMap<>();
- }
-
- /**
- * Lifecycle method to indicate end of aggregation cycle.
- */
- public void cleanup() {
- LOG.debug("Cleanup aggregated data.");
- aggregateClusterMetrics = null;
- }
-
- /**
- * Calculate aggregates if the clusterMetric is a Host metric for recorded
- * apps that are housed by this host.
- *
- * @param clusterMetric @TimelineClusterMetric Host / App metric
- * @param hostname This is the hostname from which this clusterMetric originated.
- * @param metricValue The metric value for this metric.
- */
- public void processTimelineClusterMetric(TimelineClusterMetric clusterMetric,
- String hostname, Double metricValue) {
-
- String appId = clusterMetric.getAppId();
- if (appId == null) {
- return; // No real use case except tests
- }
-
- // If metric is a host metric and host has apps on it
- if (appId.equalsIgnoreCase(HOST_APP_ID)) {
- // Candidate metric, update app aggregates
- if (hostMetadata.containsKey(hostname)) {
- updateAppAggregatesFromHostMetric(clusterMetric, hostname, metricValue);
- }
- } else {
- // Build the hostedapps map if not a host metric
- // Check app candidacy for host aggregation
- if (appIdsToAggregate.contains(appId)) {
- TimelineMetricHostMetadata timelineMetricHostMetadata = hostMetadata.get(hostname);
- ConcurrentHashMap<String, String> appIds;
- if (timelineMetricHostMetadata == null) {
- appIds = new ConcurrentHashMap<>();
- hostMetadata.put(hostname, new TimelineMetricHostMetadata(appIds));
- } else {
- appIds = timelineMetricHostMetadata.getHostedApps();
- }
- if (!appIds.containsKey(appId)) {
- appIds.put(appId, appId);
- LOG.info("Adding appId to hosted apps: appId = " +
- clusterMetric.getAppId() + ", hostname = " + hostname);
- }
- }
- }
- }
-
- /**
- * Build a cluster app metric from a host metric
- */
- private void updateAppAggregatesFromHostMetric(TimelineClusterMetric clusterMetric,
- String hostname, Double metricValue) {
-
- if (aggregateClusterMetrics == null) {
- LOG.error("Aggregation requested without init call.");
- return;
- }
-
- TimelineMetricMetadataKey appKey = new TimelineMetricMetadataKey(clusterMetric.getMetricName(), HOST_APP_ID, clusterMetric.getInstanceId());
- ConcurrentHashMap<String, String> apps = hostMetadata.get(hostname).getHostedApps();
- for (String appId : apps.keySet()) {
- if (appIdsToAggregate.contains(appId)) {
-
- appKey.setAppId(appId);
- TimelineMetricMetadata appMetadata = metadataManagerInstance.getMetadataCacheValue(appKey);
- if (appMetadata == null) {
- TimelineMetricMetadataKey key = new TimelineMetricMetadataKey(clusterMetric.getMetricName(), HOST_APP_ID, clusterMetric.getInstanceId());
- TimelineMetricMetadata hostMetricMetadata = metadataManagerInstance.getMetadataCacheValue(key);
-
- if (hostMetricMetadata != null) {
- TimelineMetricMetadata timelineMetricMetadata = new TimelineMetricMetadata(clusterMetric.getMetricName(),
- appId, clusterMetric.getInstanceId(), hostMetricMetadata.getUnits(), hostMetricMetadata.getType(), hostMetricMetadata.getSeriesStartTime(),
- hostMetricMetadata.isSupportsAggregates(), TimelineMetricsFilter.acceptMetric(clusterMetric.getMetricName(), appId));
- metadataManagerInstance.putIfModifiedTimelineMetricMetadata(timelineMetricMetadata);
- }
- }
-
- // Add a new cluster aggregate metric if none exists
- TimelineClusterMetric appTimelineClusterMetric =
- new TimelineClusterMetric(clusterMetric.getMetricName(),
- appId,
- clusterMetric.getInstanceId(),
- clusterMetric.getTimestamp());
-
- MetricClusterAggregate clusterAggregate = aggregateClusterMetrics.get(appTimelineClusterMetric);
-
- if (clusterAggregate == null) {
- clusterAggregate = new MetricClusterAggregate(metricValue, 1, null, metricValue, metricValue);
- aggregateClusterMetrics.put(appTimelineClusterMetric, clusterAggregate);
- } else {
- clusterAggregate.updateSum(metricValue);
- clusterAggregate.updateNumberOfHosts(1);
- clusterAggregate.updateMax(metricValue);
- clusterAggregate.updateMin(metricValue);
- }
- }
-
- }
- }
-
- /**
- * Return current copy of aggregated data.
- */
- public Map<TimelineClusterMetric, MetricClusterAggregate> getAggregateClusterMetrics() {
- return aggregateClusterMetrics;
- }
-
- private List<String> getAppIdsForHostAggregation(Configuration metricsConf) {
- String appIds = metricsConf.get(CLUSTER_AGGREGATOR_APP_IDS);
- if (!StringUtils.isEmpty(appIds)) {
- return Arrays.asList(StringUtils.stripAll(appIds.split(",")));
- }
- return Collections.emptyList();
- }
-}
[04/28] ambari git commit: AMBARI-22740 : Fix integration test for
HBase in branch-3.0-ams due to UUID changes. (avijayan)
Posted by av...@apache.org.
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/HBaseTimelineMetricsServiceTest.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/HBaseTimelineMetricsServiceTest.java b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/HBaseTimelineMetricsServiceTest.java
deleted file mode 100644
index e06033d..0000000
--- a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/HBaseTimelineMetricsServiceTest.java
+++ /dev/null
@@ -1,136 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline;
-
-import com.google.common.collect.Multimap;
-import junit.framework.Assert;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.aggregators.Function;
-import org.junit.Test;
-
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.TreeMap;
-
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.aggregators.Function.ReadFunction.AVG;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.aggregators.Function.ReadFunction.SUM;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.aggregators.Function.PostProcessingFunction.RATE;
-import static org.assertj.core.api.Assertions.assertThat;
-
-public class HBaseTimelineMetricsServiceTest {
-
- public static final String MEM_METRIC = "mem";
- public static final String BYTES_IN_METRIC = "bytes_in";
- public static final String BYTES_NOT_AFUNCTION_METRIC = "bytes._not._afunction";
-
- @Test
- public void testParseMetricNamesToAggregationFunctions() throws Exception {
- //giwen
- List<String> metricNames = Arrays.asList(
- MEM_METRIC + "._avg",
- MEM_METRIC + "._sum",
- MEM_METRIC + "._rate._avg",
- BYTES_IN_METRIC,
- BYTES_NOT_AFUNCTION_METRIC);
-
- //when
- Multimap<String, List<Function>> multimap =
- HBaseTimelineMetricsService.parseMetricNamesToAggregationFunctions(metricNames);
-
- //then
- Assert.assertEquals(multimap.keySet().size(), 3);
- Assert.assertTrue(multimap.containsKey(MEM_METRIC));
- Assert.assertTrue(multimap.containsKey(BYTES_IN_METRIC));
- Assert.assertTrue(multimap.containsKey(BYTES_NOT_AFUNCTION_METRIC));
-
- List<List<Function>> metricEntry = (List<List<Function>>) multimap.get(MEM_METRIC);
- HashMap<String, List<Function>> mfm = new HashMap<String, List<Function>>();
- mfm.put(MEM_METRIC, metricEntry.get(0));
-
- assertThat(mfm.get(MEM_METRIC)).containsOnly(
- new Function(AVG, null));
-
- mfm = new HashMap<String, List<Function>>();
- mfm.put(MEM_METRIC, metricEntry.get(1));
- assertThat(mfm.get(MEM_METRIC)).containsOnly(
- new Function(SUM, null));
-
- mfm = new HashMap<String, List<Function>>();
- mfm.put(MEM_METRIC, metricEntry.get(2));
- assertThat(mfm.get(MEM_METRIC)).containsOnly(
- new Function(AVG, RATE));
-
- metricEntry = (List<List<Function>>) multimap.get(BYTES_IN_METRIC);
- mfm = new HashMap<String, List<Function>>();
- mfm.put(BYTES_IN_METRIC, metricEntry.get(0));
-
- assertThat(mfm.get(BYTES_IN_METRIC))
- .contains(Function.DEFAULT_VALUE_FUNCTION);
-
- metricEntry = (List<List<Function>>) multimap.get(BYTES_NOT_AFUNCTION_METRIC);
- mfm = new HashMap<String, List<Function>>();
- mfm.put(BYTES_NOT_AFUNCTION_METRIC, metricEntry.get(0));
-
- assertThat(mfm.get(BYTES_NOT_AFUNCTION_METRIC))
- .contains(Function.DEFAULT_VALUE_FUNCTION);
-
- }
-
- @Test
- public void testRateCalculationOnMetricsWithEqualValues() throws Exception {
- Map<Long, Double> metricValues = new TreeMap<>();
- metricValues.put(1454000000000L, 1.0);
- metricValues.put(1454000001000L, 6.0);
- metricValues.put(1454000002000L, 0.0);
- metricValues.put(1454000003000L, 3.0);
- metricValues.put(1454000004000L, 4.0);
- metricValues.put(1454000005000L, 7.0);
-
- // Calculate rate
- Map<Long, Double> rates = HBaseTimelineMetricsService.updateValuesAsRate(new TreeMap<>(metricValues), false);
-
- // Make sure rate is zero
- Assert.assertTrue(rates.size() == 4);
-
- Assert.assertFalse(rates.containsKey(1454000000000L));
- Assert.assertFalse(rates.containsKey(1454000002000L));
-
- Assert.assertEquals(rates.get(1454000001000L), 5.0);
- Assert.assertEquals(rates.get(1454000003000L), 3.0);
- Assert.assertEquals(rates.get(1454000004000L), 1.0);
- Assert.assertEquals(rates.get(1454000005000L), 3.0);
- }
-
- @Test
- public void testDiffCalculation() throws Exception {
- Map<Long, Double> metricValues = new TreeMap<>();
- metricValues.put(1454016368371L, 1011.25);
- metricValues.put(1454016428371L, 1010.25);
- metricValues.put(1454016488371L, 1012.25);
- metricValues.put(1454016548371L, 1015.25);
- metricValues.put(1454016608371L, 1020.25);
-
- Map<Long, Double> rates = HBaseTimelineMetricsService.updateValuesAsRate(new TreeMap<>(metricValues), true);
-
- Assert.assertTrue(rates.size() == 3);
- Assert.assertTrue(rates.containsValue(2.0));
- Assert.assertTrue(rates.containsValue(3.0));
- Assert.assertTrue(rates.containsValue(5.0));
- }
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/ITPhoenixHBaseAccessor.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/ITPhoenixHBaseAccessor.java b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/ITPhoenixHBaseAccessor.java
deleted file mode 100644
index 2a5dd0b..0000000
--- a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/ITPhoenixHBaseAccessor.java
+++ /dev/null
@@ -1,439 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline;
-
-import static junit.framework.Assert.assertEquals;
-import static junit.framework.Assert.assertTrue;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.MetricTestHelper.createEmptyTimelineClusterMetric;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.MetricTestHelper.createEmptyTimelineMetric;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.MetricTestHelper.createMetricHostAggregate;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.MetricTestHelper.prepareSingleTimelineMetric;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.PhoenixHBaseAccessor.DATE_TIERED_COMPACTION_POLICY;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.PhoenixHBaseAccessor.FIFO_COMPACTION_POLICY_CLASS;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.PhoenixHBaseAccessor.HSTORE_COMPACTION_CLASS_KEY;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.PhoenixHBaseAccessor.HSTORE_ENGINE_CLASS;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query.PhoenixTransactSQL.METRICS_AGGREGATE_MINUTE_TABLE_NAME;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query.PhoenixTransactSQL.METRICS_RECORD_TABLE_NAME;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query.PhoenixTransactSQL.PHOENIX_TABLES;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query.PhoenixTransactSQL.PHOENIX_TABLES_REGEX_PATTERN;
-
-import java.io.IOException;
-import java.lang.reflect.Field;
-import java.sql.PreparedStatement;
-import java.sql.ResultSet;
-import java.sql.SQLException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.Optional;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.Admin;
-import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
-import org.apache.hadoop.hbase.client.Durability;
-import org.apache.hadoop.hbase.client.TableDescriptor;
-import org.apache.hadoop.metrics2.sink.timeline.ContainerMetric;
-import org.apache.hadoop.metrics2.sink.timeline.MetricClusterAggregate;
-import org.apache.hadoop.metrics2.sink.timeline.MetricHostAggregate;
-import org.apache.hadoop.metrics2.sink.timeline.Precision;
-import org.apache.hadoop.metrics2.sink.timeline.TimelineMetric;
-import org.apache.hadoop.metrics2.sink.timeline.TimelineMetrics;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.aggregators.Function;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.aggregators.TimelineClusterMetric;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.aggregators.TimelineMetricAggregator;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.aggregators.TimelineMetricAggregatorFactory;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.discovery.TimelineMetricMetadataManager;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query.Condition;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query.DefaultCondition;
-import org.junit.Test;
-
-import com.google.common.collect.ArrayListMultimap;
-import com.google.common.collect.Multimap;
-
-import junit.framework.Assert;
-
-
-
-public class ITPhoenixHBaseAccessor extends AbstractMiniHBaseClusterTest {
-
- @Test
- public void testGetMetricRecordsSeconds() throws IOException, SQLException {
- // GIVEN
- long startTime = System.currentTimeMillis();
- long ctime = startTime;
- long minute = 60 * 1000;
- hdb.insertMetricRecords(prepareSingleTimelineMetric(ctime, "local1",
- "disk_free", 1));
- hdb.insertMetricRecords(prepareSingleTimelineMetric(ctime, "local2",
- "disk_free", 2));
- ctime += minute;
- hdb.insertMetricRecords(prepareSingleTimelineMetric(ctime, "local1",
- "disk_free", 2));
- hdb.insertMetricRecords(prepareSingleTimelineMetric(ctime, "local2",
- "disk_free", 1));
-
- // WHEN
- long endTime = ctime + minute;
- Condition condition = new DefaultCondition(
- new ArrayList<String>() {{ add("disk_free"); }},
- Collections.singletonList("local1"),
- null, null, startTime, endTime, Precision.SECONDS, null, true);
- TimelineMetrics timelineMetrics = hdb.getMetricRecords(condition,
- singletonValueFunctionMap("disk_free"));
-
- //THEN
- assertEquals(1, timelineMetrics.getMetrics().size());
- TimelineMetric metric = timelineMetrics.getMetrics().get(0);
-
- assertEquals("disk_free", metric.getMetricName());
- assertEquals("local1", metric.getHostName());
- assertEquals(8, metric.getMetricValues().size());
- }
-
- @Test
- public void testGetMetricRecordsMinutes() throws IOException, SQLException {
- // GIVEN
- TimelineMetricAggregator aggregatorMinute =
- TimelineMetricAggregatorFactory.createTimelineMetricAggregatorMinute(hdb, new Configuration(), null, null);
-
- long startTime = System.currentTimeMillis();
- long ctime = startTime;
- long minute = 60 * 1000;
- hdb.insertMetricRecords(prepareSingleTimelineMetric(ctime, "local1",
- "disk_free", 1));
- hdb.insertMetricRecords(prepareSingleTimelineMetric(ctime + minute, "local1",
- "disk_free", 2));
- hdb.insertMetricRecords(prepareSingleTimelineMetric(ctime, "local2",
- "disk_free", 2));
- long endTime = ctime + minute;
- boolean success = aggregatorMinute.doWork(startTime, endTime);
- assertTrue(success);
-
- // WHEN
- Condition condition = new DefaultCondition(
- new ArrayList<String>() {{ add("disk_free"); }},
- Collections.singletonList("local1"),
- null, null, startTime, endTime, Precision.MINUTES, null, false);
- TimelineMetrics timelineMetrics = hdb.getMetricRecords(condition,
- singletonValueFunctionMap("disk_free"));
-
- //THEN
- assertEquals(1, timelineMetrics.getMetrics().size());
- TimelineMetric metric = timelineMetrics.getMetrics().get(0);
-
- assertEquals("disk_free", metric.getMetricName());
- assertEquals("local1", metric.getHostName());
- assertEquals(1, metric.getMetricValues().size());
- Iterator<Map.Entry<Long, Double>> iterator = metric.getMetricValues().entrySet().iterator();
- assertEquals(1.5, iterator.next().getValue(), 0.00001);
- }
-
- @Test
- public void testGetMetricRecordsHours() throws IOException, SQLException {
- // GIVEN
- TimelineMetricAggregator aggregator =
- TimelineMetricAggregatorFactory.createTimelineMetricAggregatorHourly(hdb, new Configuration(), null, null);
-
- MetricHostAggregate expectedAggregate =
- createMetricHostAggregate(2.0, 0.0, 20, 15.0);
- Map<TimelineMetric, MetricHostAggregate>
- aggMap = new HashMap<TimelineMetric,
- MetricHostAggregate>();
-
- long startTime = System.currentTimeMillis();
- int min_5 = 5 * 60 * 1000;
- long ctime = startTime - min_5;
- aggMap.put(createEmptyTimelineMetric(ctime += min_5), expectedAggregate);
- aggMap.put(createEmptyTimelineMetric(ctime += min_5), expectedAggregate);
- aggMap.put(createEmptyTimelineMetric(ctime += min_5), expectedAggregate);
- aggMap.put(createEmptyTimelineMetric(ctime += min_5), expectedAggregate);
- aggMap.put(createEmptyTimelineMetric(ctime += min_5), expectedAggregate);
- aggMap.put(createEmptyTimelineMetric(ctime += min_5), expectedAggregate);
- aggMap.put(createEmptyTimelineMetric(ctime += min_5), expectedAggregate);
- aggMap.put(createEmptyTimelineMetric(ctime += min_5), expectedAggregate);
- aggMap.put(createEmptyTimelineMetric(ctime += min_5), expectedAggregate);
- aggMap.put(createEmptyTimelineMetric(ctime += min_5), expectedAggregate);
- aggMap.put(createEmptyTimelineMetric(ctime += min_5), expectedAggregate);
- aggMap.put(createEmptyTimelineMetric(ctime += min_5), expectedAggregate);
-
- hdb.saveHostAggregateRecords(aggMap, METRICS_AGGREGATE_MINUTE_TABLE_NAME);
- long endTime = ctime + min_5;
- boolean success = aggregator.doWork(startTime, endTime);
- assertTrue(success);
-
- // WHEN
- Condition condition = new DefaultCondition(
- new ArrayList<String>() {{ add("disk_free"); }},
- Collections.singletonList("test_host"),
- "test_app", null, startTime, endTime, Precision.HOURS, null, true);
- TimelineMetrics timelineMetrics = hdb.getMetricRecords(condition,
- singletonValueFunctionMap("disk_used"));
-
- //THEN
- assertEquals(1, timelineMetrics.getMetrics().size());
- TimelineMetric metric = timelineMetrics.getMetrics().get(0);
-
- assertEquals("disk_used", metric.getMetricName());
- assertEquals("test_host", metric.getHostName());
- assertEquals(1, metric.getMetricValues().size());
- Iterator<Map.Entry<Long, Double>> iterator = metric.getMetricValues().entrySet().iterator();
- assertEquals(0.75, iterator.next().getValue(), 0.00001);
- }
-
- @Test
- public void testGetClusterMetricRecordsSeconds() throws Exception {
- // GIVEN
- TimelineMetricAggregator agg =
- TimelineMetricAggregatorFactory.createTimelineClusterAggregatorSecond(
- hdb, new Configuration(), new TimelineMetricMetadataManager(new Configuration(), hdb), null, null);
-
- long startTime = System.currentTimeMillis();
- long ctime = startTime + 1;
- long minute = 60 * 1000;
- hdb.insertMetricRecords(prepareSingleTimelineMetric(ctime, "local1",
- "disk_free", 1));
- hdb.insertMetricRecords(prepareSingleTimelineMetric(ctime, "local2",
- "disk_free", 2));
- ctime += minute;
- hdb.insertMetricRecords(prepareSingleTimelineMetric(ctime, "local1",
- "disk_free", 2));
- hdb.insertMetricRecords(prepareSingleTimelineMetric(ctime, "local2",
- "disk_free", 1));
-
- long endTime = ctime + minute + 1;
- boolean success = agg.doWork(startTime, endTime);
- assertTrue(success);
-
- // WHEN
- Condition condition = new DefaultCondition(
- new ArrayList<String>() {{ add("disk_free"); }},
- null, null, null, startTime, endTime, Precision.SECONDS, null, true);
- TimelineMetrics timelineMetrics = hdb.getAggregateMetricRecords(condition,
- singletonValueFunctionMap("disk_free"));
-
- //THEN
- assertEquals(1, timelineMetrics.getMetrics().size());
- TimelineMetric metric = timelineMetrics.getMetrics().get(0);
-
- assertEquals("disk_free", metric.getMetricName());
- assertEquals(5, metric.getMetricValues().size());
- assertEquals(1.5, metric.getMetricValues().values().iterator().next(), 0.00001);
- }
-
- @Test
- public void testGetClusterMetricRecordLatestWithFunction() throws Exception {
- // GIVEN
- TimelineMetricAggregator agg =
- TimelineMetricAggregatorFactory.createTimelineClusterAggregatorSecond(hdb,
- new Configuration(), new TimelineMetricMetadataManager(new Configuration(), hdb), null, null);
-
- long startTime = System.currentTimeMillis();
- long ctime = startTime + 1;
- long minute = 60 * 1000;
- hdb.insertMetricRecords(prepareSingleTimelineMetric(ctime, "local1",
- "disk_free", 1));
- hdb.insertMetricRecords(prepareSingleTimelineMetric(ctime, "local2",
- "disk_free", 2));
- ctime += minute;
- hdb.insertMetricRecords(prepareSingleTimelineMetric(ctime, "local1",
- "disk_free", 2));
- hdb.insertMetricRecords(prepareSingleTimelineMetric(ctime, "local2",
- "disk_free", 1));
-
- long endTime = ctime + minute + 1;
- boolean success = agg.doWork(startTime, endTime);
- assertTrue(success);
-
- // WHEN
- Condition condition = new DefaultCondition(
- new ArrayList<String>() {{ add("disk_free"); }},
- null, null, null, null, null, Precision.SECONDS, null, true);
-
- Multimap<String, List<Function>> mmap = ArrayListMultimap.create();
- mmap.put("disk_free", Collections.singletonList(new Function(Function.ReadFunction.SUM, null)));
- TimelineMetrics timelineMetrics = hdb.getAggregateMetricRecords(condition, mmap);
-
- //THEN
- assertEquals(1, timelineMetrics.getMetrics().size());
- TimelineMetric metric = timelineMetrics.getMetrics().get(0);
-
- assertEquals("disk_free._sum", metric.getMetricName());
- assertEquals(1, metric.getMetricValues().size());
- assertEquals(3, metric.getMetricValues().values().iterator().next().intValue());
- }
-
- @Test
- public void testGetClusterMetricRecordsHours() throws Exception {
- // GIVEN
- TimelineMetricAggregator agg =
- TimelineMetricAggregatorFactory.createTimelineClusterAggregatorHourly(hdb, new Configuration(), null, null);
-
- long startTime = System.currentTimeMillis();
- long ctime = startTime;
- long minute = 60 * 1000;
-
- Map<TimelineClusterMetric, MetricClusterAggregate> records =
- new HashMap<TimelineClusterMetric, MetricClusterAggregate>();
-
- records.put(createEmptyTimelineClusterMetric(ctime),
- new MetricClusterAggregate(4.0, 2, 0.0, 4.0, 0.0));
- records.put(createEmptyTimelineClusterMetric(ctime += minute),
- new MetricClusterAggregate(4.0, 2, 0.0, 4.0, 0.0));
- records.put(createEmptyTimelineClusterMetric(ctime += minute),
- new MetricClusterAggregate(4.0, 2, 0.0, 4.0, 0.0));
- records.put(createEmptyTimelineClusterMetric(ctime += minute),
- new MetricClusterAggregate(4.0, 2, 0.0, 4.0, 0.0));
-
- hdb.saveClusterAggregateRecords(records);
- boolean success = agg.doWork(startTime, ctime + minute);
- assertTrue(success);
-
- // WHEN
- Condition condition = new DefaultCondition(
- new ArrayList<String>() {{ add("disk_free"); }},
- null, null, null, startTime, ctime + minute, Precision.HOURS, null, true);
- TimelineMetrics timelineMetrics = hdb.getAggregateMetricRecords(condition,
- singletonValueFunctionMap("disk_used"));
-
- // THEN
- assertEquals(1, timelineMetrics.getMetrics().size());
- TimelineMetric metric = timelineMetrics.getMetrics().get(0);
-
- assertEquals("disk_used", metric.getMetricName());
- assertEquals("test_app", metric.getAppId());
- assertEquals(1, metric.getMetricValues().size());
- assertEquals(2.0, metric.getMetricValues().values().iterator().next(), 0.00001);
- }
-
- @Test
- public void testInitPoliciesAndTTL() throws Exception {
- Admin hBaseAdmin = hdb.getHBaseAdmin();
- int precisionTtl = 2 * 86400;
-
- Field f = PhoenixHBaseAccessor.class.getDeclaredField("tableTTL");
- f.setAccessible(true);
- Map<String, Integer> precisionValues = (Map<String, Integer>) f.get(hdb);
- precisionValues.put(METRICS_RECORD_TABLE_NAME, precisionTtl);
- f.set(hdb, precisionValues);
-
- Field f2 = PhoenixHBaseAccessor.class.getDeclaredField("timelineMetricsTablesDurability");
- f2.setAccessible(true);
- f2.set(hdb, "ASYNC_WAL");
-
- hdb.initPoliciesAndTTL();
-
- // Verify expected policies are set
- boolean normalizerEnabled = false;
- String precisionTableCompactionPolicy = null;
- String aggregateTableCompactionPolicy = null;
- boolean tableDurabilitySet = false;
- for (int i = 0; i < 10; i++) {
- LOG.warn("Policy check retry : " + i);
- for (String tableName : PHOENIX_TABLES) {
- TableName[] tableNames = hBaseAdmin.listTableNames(PHOENIX_TABLES_REGEX_PATTERN, false);
- Optional<TableName> tableNameOptional = Arrays.stream(tableNames)
- .filter(t -> tableName.equals(t.getNameAsString())).findFirst();
-
- TableDescriptor tableDescriptor = hBaseAdmin.getTableDescriptor(tableNameOptional.get());
-
- normalizerEnabled = tableDescriptor.isNormalizationEnabled();
- tableDurabilitySet = (Durability.ASYNC_WAL.equals(tableDescriptor.getDurability()));
- if (tableName.equals(METRICS_RECORD_TABLE_NAME)) {
- precisionTableCompactionPolicy = tableDescriptor.getValue(HSTORE_ENGINE_CLASS);
- } else {
- aggregateTableCompactionPolicy = tableDescriptor.getValue(HSTORE_COMPACTION_CLASS_KEY);
- }
- LOG.debug("Table: " + tableName + ", normalizerEnabled = " + normalizerEnabled);
- // Best effort for 20 seconds
- if (normalizerEnabled || (precisionTableCompactionPolicy == null && aggregateTableCompactionPolicy ==null)) {
- Thread.sleep(20000l);
- }
- if (tableName.equals(METRICS_RECORD_TABLE_NAME)) {
- for (ColumnFamilyDescriptor family : tableDescriptor.getColumnFamilies()) {
- precisionTtl = family.getTimeToLive();
- }
- }
- }
- }
-
- Assert.assertFalse("Normalizer disabled.", normalizerEnabled);
- Assert.assertTrue("Durability Set.", tableDurabilitySet);
- Assert.assertEquals("FIFO compaction policy is set for METRIC_RECORD.", FIFO_COMPACTION_POLICY_CLASS, precisionTableCompactionPolicy);
- Assert.assertEquals("FIFO compaction policy is set for aggregate tables", DATE_TIERED_COMPACTION_POLICY, aggregateTableCompactionPolicy);
- Assert.assertEquals("Precision TTL value as expected.", 2 * 86400, precisionTtl);
-
- hBaseAdmin.close();
- }
-
- private Multimap<String, List<Function>> singletonValueFunctionMap(String metricName) {
- Multimap<String, List<Function>> mmap = ArrayListMultimap.create();
- mmap.put(metricName, Collections.singletonList(new Function()));
- return mmap;
- }
-
- @Test
- public void testInsertContainerMetrics() throws Exception {
- ContainerMetric metric = new ContainerMetric();
- metric.setContainerId("container_1450744875949_0001_01_000001");
- metric.setHostName("host1");
- metric.setPmemLimit(2048);
- metric.setVmemLimit(2048);
- metric.setPmemUsedAvg(1024);
- metric.setPmemUsedMin(1024);
- metric.setPmemUsedMax(1024);
- metric.setLaunchDuration(2000);
- metric.setLocalizationDuration(3000);
- long startTime = System.currentTimeMillis();
- long finishTime = startTime + 5000;
- metric.setStartTime(startTime);
- metric.setFinishTime(finishTime);
- metric.setExitCode(0);
- List<ContainerMetric> list = Arrays.asList(metric);
- hdb.insertContainerMetrics(list);
- PreparedStatement stmt = conn.prepareStatement("SELECT * FROM CONTAINER_METRICS");
- ResultSet set = stmt.executeQuery();
- // check each filed is set properly when read back.
- boolean foundRecord = false;
- while (set.next()) {
- assertEquals("application_1450744875949_0001", set.getString("APP_ID"));
- assertEquals("container_1450744875949_0001_01_000001", set.getString("CONTAINER_ID"));
- assertEquals(new java.sql.Timestamp(startTime), set.getTimestamp("SERVER_TIME"));
- assertEquals(new java.sql.Timestamp(finishTime), set.getTimestamp("FINISH_TIME"));
- assertEquals(5000, set.getLong("DURATION"));
- assertEquals("host1", set.getString("HOSTNAME"));
- assertEquals(0, set.getInt("EXIT_CODE"));
- assertEquals(3000, set.getLong("LOCALIZATION_DURATION"));
- assertEquals(2000, set.getLong("LAUNCH_DURATION"));
- assertEquals((double)2, set.getDouble("MEM_REQUESTED_GB"));
- assertEquals((double)2 * 5000, set.getDouble("MEM_REQUESTED_GB_MILLIS"));
- assertEquals((double)2, set.getDouble("MEM_VIRTUAL_GB"));
- assertEquals((double)1, set.getDouble("MEM_USED_GB_MIN"));
- assertEquals((double)1, set.getDouble("MEM_USED_GB_MAX"));
- assertEquals((double)1, set.getDouble("MEM_USED_GB_AVG"));
- assertEquals((double)(2 - 1), set.getDouble("MEM_UNUSED_GB"));
- assertEquals((double)(2-1) * 5000, set.getDouble("MEM_UNUSED_GB_MILLIS"));
- foundRecord = true;
- }
- assertTrue(foundRecord);
- }
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/MetricTestHelper.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/MetricTestHelper.java b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/MetricTestHelper.java
deleted file mode 100644
index 74da438..0000000
--- a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/MetricTestHelper.java
+++ /dev/null
@@ -1,121 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline;
-
-import org.apache.hadoop.metrics2.sink.timeline.MetricHostAggregate;
-import org.apache.hadoop.metrics2.sink.timeline.TimelineMetric;
-import org.apache.hadoop.metrics2.sink.timeline.TimelineMetrics;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.aggregators.TimelineClusterMetric;
-
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.TreeMap;
-
-public class MetricTestHelper {
-
- public static MetricHostAggregate createMetricHostAggregate(double max, double min, int numberOfSamples, double sum) {
- MetricHostAggregate expectedAggregate = new MetricHostAggregate();
- expectedAggregate.setMax(max);
- expectedAggregate.setMin(min);
- expectedAggregate.setNumberOfSamples(numberOfSamples);
- expectedAggregate.setSum(sum);
-
- return expectedAggregate;
- }
-
- public static TimelineMetrics prepareSingleTimelineMetric(long startTime,
- String host,
- String metricName,
- double val) {
- return prepareSingleTimelineMetric(startTime, host, null, metricName, val);
- }
-
- public static TimelineMetrics prepareSingleTimelineMetric(long startTime,
- String host,
- String instanceId,
- String metricName,
- double val) {
- TimelineMetrics m = new TimelineMetrics();
- m.setMetrics(Arrays.asList(
- createTimelineMetric(startTime, metricName, host, null, instanceId, val)));
-
- return m;
- }
-
- public static TimelineMetrics prepareSingleTimelineMetric(long startTime,
- String host,
- String appId,
- String instanceId,
- String metricName,
- double val) {
- TimelineMetrics m = new TimelineMetrics();
- m.setMetrics(Arrays.asList(
- createTimelineMetric(startTime, metricName, host, appId, instanceId, val)));
-
- return m;
- }
-
-
- public static TimelineMetric createTimelineMetric(long startTime,
- String metricName,
- String host,
- String appId,
- String instanceId,
- double val) {
- TimelineMetric m = new TimelineMetric();
- m.setHostName(host);
- m.setAppId(appId != null ? appId : "host");
- m.setInstanceId(instanceId);
- m.setMetricName(metricName);
- m.setStartTime(startTime);
- TreeMap<Long, Double> vals = new TreeMap<Long, Double>();
- vals.put(startTime + 15000l, val);
- vals.put(startTime + 30000l, val);
- vals.put(startTime + 45000l, val);
- vals.put(startTime + 60000l, val);
-
- m.setMetricValues(vals);
-
- return m;
- }
-
- public static TimelineMetric createEmptyTimelineMetric(long startTime) {
- TimelineMetric metric = new TimelineMetric();
- metric.setMetricName("disk_used");
- metric.setAppId("test_app");
- metric.setInstanceId("test_instance");
- metric.setHostName("test_host");
- metric.setStartTime(startTime);
-
- return metric;
- }
-
- public static TimelineClusterMetric createEmptyTimelineClusterMetric(
- String name, long startTime) {
- TimelineClusterMetric metric = new TimelineClusterMetric(name,
- "test_app", "instance_id", startTime);
-
- return metric;
- }
-
- public static TimelineClusterMetric createEmptyTimelineClusterMetric(
- long startTime) {
- return createEmptyTimelineClusterMetric("disk_used", startTime);
- }
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/PhoenixHBaseAccessorTest.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/PhoenixHBaseAccessorTest.java b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/PhoenixHBaseAccessorTest.java
deleted file mode 100644
index 50ff656..0000000
--- a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/PhoenixHBaseAccessorTest.java
+++ /dev/null
@@ -1,292 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline;
-
-import com.google.common.collect.ArrayListMultimap;
-import com.google.common.collect.Multimap;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.DoNotRetryIOException;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
-import org.apache.hadoop.hbase.util.RetryCounterFactory;
-import org.apache.hadoop.metrics2.sink.timeline.MetricClusterAggregate;
-import org.apache.hadoop.metrics2.sink.timeline.MetricHostAggregate;
-import org.apache.hadoop.metrics2.sink.timeline.Precision;
-import org.apache.hadoop.metrics2.sink.timeline.TimelineMetric;
-import org.apache.hadoop.metrics2.sink.timeline.TimelineMetrics;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.aggregators.TimelineClusterMetric;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.aggregators.Function;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.discovery.TimelineMetricMetadataManager;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query.Condition;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query.DefaultCondition;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query.PhoenixConnectionProvider;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query.PhoenixTransactSQL;
-import org.apache.phoenix.exception.PhoenixIOException;
-import org.easymock.EasyMock;
-import org.junit.Before;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.powermock.core.classloader.annotations.PrepareForTest;
-import org.powermock.modules.junit4.PowerMockRunner;
-
-import java.io.IOException;
-import java.sql.Connection;
-import java.sql.PreparedStatement;
-import java.sql.ResultSet;
-import java.sql.SQLException;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-
-import static org.easymock.EasyMock.anyObject;
-import static org.easymock.EasyMock.expect;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.fail;
-import static org.powermock.api.easymock.PowerMock.*;
-
-@RunWith(PowerMockRunner.class)
-@PrepareForTest({PhoenixTransactSQL.class, TimelineMetricConfiguration.class})
-public class PhoenixHBaseAccessorTest {
- private static final String ZOOKEEPER_QUORUM = "hbase.zookeeper.quorum";
-
- PhoenixConnectionProvider connectionProvider;
- PhoenixHBaseAccessor accessor;
-
- @Before
- public void setupConf() throws Exception {
- Configuration hbaseConf = new Configuration();
- hbaseConf.setStrings(ZOOKEEPER_QUORUM, "quorum");
- Configuration metricsConf = new Configuration();
- metricsConf.setStrings(TimelineMetricConfiguration.TIMELINE_METRICS_CACHE_SIZE, "1");
- metricsConf.setStrings(TimelineMetricConfiguration.TIMELINE_METRICS_CACHE_COMMIT_INTERVAL, "100");
- metricsConf.setStrings(
- TimelineMetricConfiguration.TIMELINE_METRIC_AGGREGATOR_SINK_CLASS,
- "org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricsAggregatorMemorySink");
-
- TimelineMetricConfiguration conf = new TimelineMetricConfiguration(hbaseConf, metricsConf);
- mockStatic(TimelineMetricConfiguration.class);
- expect(TimelineMetricConfiguration.getInstance()).andReturn(conf).anyTimes();
- replayAll();
-
- connectionProvider = new PhoenixConnectionProvider() {
- @Override
- public HBaseAdmin getHBaseAdmin() throws IOException {
- return null;
- }
-
- @Override
- public Connection getConnection() throws SQLException {
- return null;
- }
-
- };
-
- accessor = new PhoenixHBaseAccessor(connectionProvider);
- }
-
- @Test
- public void testGetMetricRecords() throws SQLException, IOException {
- List<String> metricNames = new LinkedList<>();
- List<String> hostnames = new LinkedList<>();
- Multimap<String, List<Function>> metricFunctions = ArrayListMultimap.create();
-
- mockStatic(PhoenixTransactSQL.class);
- PreparedStatement preparedStatementMock = EasyMock.createNiceMock(PreparedStatement.class);
- Condition condition = new DefaultCondition(metricNames, hostnames, "appid", "instanceid", 123L, 234L, Precision.SECONDS, 10, true);
- expect(PhoenixTransactSQL.prepareGetMetricsSqlStmt(null, condition)).andReturn(preparedStatementMock).once();
- ResultSet rsMock = EasyMock.createNiceMock(ResultSet.class);
- expect(preparedStatementMock.executeQuery()).andReturn(rsMock);
-
-
- replayAll();
- EasyMock.replay(preparedStatementMock, rsMock);
-
- // Check when startTime < endTime
- TimelineMetrics tml = accessor.getMetricRecords(condition, metricFunctions);
-
- // Check when startTime > endTime
- Condition condition2 = new DefaultCondition(metricNames, hostnames, "appid", "instanceid", 234L, 123L, Precision.SECONDS, 10, true);
- TimelineMetrics tml2 = accessor.getMetricRecords(condition2, metricFunctions);
- assertEquals(0, tml2.getMetrics().size());
-
- verifyAll();
- EasyMock.verify(preparedStatementMock, rsMock);
- }
-
- @Test
- public void testGetMetricRecordsIOException() throws SQLException, IOException {
- List<String> metricNames = new LinkedList<>();
- List<String> hostnames = new LinkedList<>();
- Multimap<String, List<Function>> metricFunctions = ArrayListMultimap.create();
-
- mockStatic(PhoenixTransactSQL.class);
- PreparedStatement preparedStatementMock = EasyMock.createNiceMock(PreparedStatement.class);
- Condition condition = new DefaultCondition(metricNames, hostnames, "appid", "instanceid", 123L, 234L, Precision.SECONDS, 10, true);
- expect(PhoenixTransactSQL.prepareGetMetricsSqlStmt(null, condition)).andReturn(preparedStatementMock).once();
- ResultSet rsMock = EasyMock.createNiceMock(ResultSet.class);
- RuntimeException runtimeException = EasyMock.createNiceMock(RuntimeException.class);
- IOException io = EasyMock.createNiceMock(IOException.class);
- expect(preparedStatementMock.executeQuery()).andThrow(runtimeException);
- expect(runtimeException.getCause()).andReturn(io).atLeastOnce();
- StackTraceElement stackTrace[] = new StackTraceElement[]{new StackTraceElement("TimeRange","method","file",1)};
- expect(io.getStackTrace()).andReturn(stackTrace).atLeastOnce();
-
-
- replayAll();
- EasyMock.replay(preparedStatementMock, rsMock, io, runtimeException);
-
- TimelineMetrics tml = accessor.getMetricRecords(condition, metricFunctions);
-
- assertEquals(0, tml.getMetrics().size());
-
- verifyAll();
- EasyMock.verify(preparedStatementMock, rsMock, io, runtimeException);
- }
-
- @Test
- public void testGetMetricRecordsPhoenixIOExceptionDoNotRetryException() throws SQLException, IOException {
- List<String> metricNames = new LinkedList<>();
- List<String> hostnames = new LinkedList<>();
- Multimap<String, List<Function>> metricFunctions = ArrayListMultimap.create();
-
- mockStatic(PhoenixTransactSQL.class);
- PreparedStatement preparedStatementMock = EasyMock.createNiceMock(PreparedStatement.class);
- Condition condition = new DefaultCondition(metricNames, hostnames, "appid", "instanceid", null, null, Precision.SECONDS, 10, true);
- expect(PhoenixTransactSQL.prepareGetLatestMetricSqlStmt(null, condition)).andReturn(preparedStatementMock).once();
- PhoenixTransactSQL.setSortMergeJoinEnabled(true);
- EasyMock.expectLastCall();
- ResultSet rsMock = EasyMock.createNiceMock(ResultSet.class);
- PhoenixIOException pioe1 = EasyMock.createNiceMock(PhoenixIOException.class);
- PhoenixIOException pioe2 = EasyMock.createNiceMock(PhoenixIOException.class);
- DoNotRetryIOException dnrioe = EasyMock.createNiceMock(DoNotRetryIOException.class);
- expect(preparedStatementMock.executeQuery()).andThrow(pioe1);
- expect(pioe1.getCause()).andReturn(pioe2).atLeastOnce();
- expect(pioe2.getCause()).andReturn(dnrioe).atLeastOnce();
- StackTraceElement stackTrace[] = new StackTraceElement[]{new StackTraceElement("HashJoinRegionScanner","method","file",1)};
- expect(dnrioe.getStackTrace()).andReturn(stackTrace).atLeastOnce();
-
-
- replayAll();
- EasyMock.replay(preparedStatementMock, rsMock, pioe1, pioe2, dnrioe);
- try {
- accessor.getMetricRecords(condition, metricFunctions);
- fail();
- } catch (Exception e) {
- //NOP
- }
- verifyAll();
- }
-
- @Test
- public void testMetricsCacheCommittingWhenFull() throws IOException, SQLException {
- Configuration hbaseConf = new Configuration();
- hbaseConf.setStrings(ZOOKEEPER_QUORUM, "quorum");
-
- final Connection connection = EasyMock.createNiceMock(Connection.class);
-
- accessor = new PhoenixHBaseAccessor(connectionProvider) {
- @Override
- public void commitMetrics(Collection<TimelineMetrics> timelineMetricsCollection) {
- try {
- connection.commit();
- } catch (SQLException e) {
- e.printStackTrace();
- }
- }
- };
-
- TimelineMetrics timelineMetrics = EasyMock.createNiceMock(TimelineMetrics.class);
- expect(timelineMetrics.getMetrics()).andReturn(Collections.singletonList(new TimelineMetric())).anyTimes();
- connection.commit();
- EasyMock.expectLastCall().once();
-
- EasyMock.replay(timelineMetrics, connection);
-
- accessor.insertMetricRecords(timelineMetrics);
- accessor.insertMetricRecords(timelineMetrics);
- accessor.insertMetricRecords(timelineMetrics);
-
- EasyMock.verify(timelineMetrics, connection);
- }
-
- @Test
- public void testMetricsAggregatorSink() throws IOException, SQLException {
- Map<TimelineClusterMetric, MetricClusterAggregate> clusterAggregateMap =
- new HashMap<>();
- Map<TimelineClusterMetric, MetricHostAggregate> clusterTimeAggregateMap =
- new HashMap<>();
- Map<TimelineMetric, MetricHostAggregate> hostAggregateMap = new HashMap<>();
-
-
- final Connection connection = EasyMock.createNiceMock(Connection.class);
- final PreparedStatement statement = EasyMock.createNiceMock(PreparedStatement.class);
- expect(connection.prepareStatement(EasyMock.anyString())).andReturn(statement).anyTimes();
- EasyMock.replay(statement);
- EasyMock.replay(connection);
-
- connectionProvider = new PhoenixConnectionProvider() {
-
- @Override
- public HBaseAdmin getHBaseAdmin() throws IOException {
- return null;
- }
-
- @Override
- public Connection getConnection() throws SQLException {
- return connection;
- }
- };
-
- accessor = new PhoenixHBaseAccessor(connectionProvider);
-
- TimelineClusterMetric clusterMetric =
- new TimelineClusterMetric("metricName", "appId", "instanceId",
- System.currentTimeMillis());
- TimelineMetric timelineMetric = new TimelineMetric();
- timelineMetric.setMetricName("Metric1");
- timelineMetric.setType("type1");
- timelineMetric.setAppId("App1");
- timelineMetric.setInstanceId("instance1");
- timelineMetric.setHostName("host1");
-
- clusterAggregateMap.put(clusterMetric, new MetricClusterAggregate());
- clusterTimeAggregateMap.put(clusterMetric, new MetricHostAggregate());
- hostAggregateMap.put(timelineMetric, new MetricHostAggregate());
-
- TimelineMetricMetadataManager metricMetadataManagerMock = EasyMock.createMock(TimelineMetricMetadataManager.class);
- expect(metricMetadataManagerMock.getUuid(anyObject(TimelineClusterMetric.class))).andReturn(new byte[16]).times(2);
- expect(metricMetadataManagerMock.getUuid(anyObject(TimelineMetric.class))).andReturn(new byte[20]).once();
- replay(metricMetadataManagerMock);
-
- accessor.setMetadataInstance(metricMetadataManagerMock);
- accessor.saveClusterAggregateRecords(clusterAggregateMap);
- accessor.saveHostAggregateRecords(hostAggregateMap,
- PhoenixTransactSQL.METRICS_AGGREGATE_MINUTE_TABLE_NAME);
- accessor.saveClusterAggregateRecordsSecond(clusterTimeAggregateMap,
- PhoenixTransactSQL.METRICS_CLUSTER_AGGREGATE_HOURLY_TABLE_NAME);
-
- TimelineMetricsAggregatorMemorySink memorySink =
- new TimelineMetricsAggregatorMemorySink();
- assertEquals(1, memorySink.getClusterAggregateRecords().size());
- assertEquals(1, memorySink.getClusterTimeAggregateRecords().size());
- assertEquals(1, memorySink.getHostAggregateRecords().size());
- }
-
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TestClusterSuite.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TestClusterSuite.java b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TestClusterSuite.java
deleted file mode 100644
index 1fc5b0e..0000000
--- a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TestClusterSuite.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline;
-
-
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.aggregators.ITClusterAggregator;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.aggregators.ITMetricAggregator;
-import org.junit.Ignore;
-import org.junit.runner.RunWith;
-import org.junit.runners.Suite;
-
-import static org.junit.runners.Suite.SuiteClasses;
-
-@Ignore
-@RunWith(Suite.class)
-@SuiteClasses({ITMetricAggregator.class, ITClusterAggregator.class, ITPhoenixHBaseAccessor.class})
-public class TestClusterSuite {
-
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TestMetricHostAggregate.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TestMetricHostAggregate.java b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TestMetricHostAggregate.java
deleted file mode 100644
index 3009163..0000000
--- a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TestMetricHostAggregate.java
+++ /dev/null
@@ -1,66 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics
- .timeline;
-
-import org.apache.hadoop.metrics2.sink.timeline.MetricHostAggregate;
-import org.junit.Test;
-
-import static org.assertj.core.api.Assertions.assertThat;
-
-public class TestMetricHostAggregate {
-
- @Test
- public void testCreateAggregate() throws Exception {
- // given
- MetricHostAggregate aggregate = createAggregate(3.0, 1.0, 2.0, 2);
-
- //then
- assertThat(aggregate.getSum()).isEqualTo(3.0);
- assertThat(aggregate.getMin()).isEqualTo(1.0);
- assertThat(aggregate.getMax()).isEqualTo(2.0);
- assertThat(aggregate.calculateAverage()).isEqualTo(3.0 / 2);
- }
-
- @Test
- public void testUpdateAggregates() throws Exception {
- // given
- MetricHostAggregate aggregate = createAggregate(3.0, 1.0, 2.0, 2);
-
- //when
- aggregate.updateAggregates(createAggregate(8.0, 0.5, 7.5, 2));
- aggregate.updateAggregates(createAggregate(1.0, 1.0, 1.0, 1));
-
- //then
- assertThat(aggregate.getSum()).isEqualTo(12.0);
- assertThat(aggregate.getMin()).isEqualTo(0.5);
- assertThat(aggregate.getMax()).isEqualTo(7.5);
- assertThat(aggregate.calculateAverage()).isEqualTo((3.0 + 8.0 + 1.0) / 5);
- }
-
- static MetricHostAggregate createAggregate (Double sum, Double min,
- Double max, Integer samplesCount) {
- MetricHostAggregate aggregate = new MetricHostAggregate();
- aggregate.setSum(sum);
- aggregate.setMax(max);
- aggregate.setMin(min);
- aggregate.setDeviation(0.0);
- aggregate.setNumberOfSamples(samplesCount);
- return aggregate;
- }
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TestPhoenixTransactSQL.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TestPhoenixTransactSQL.java b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TestPhoenixTransactSQL.java
deleted file mode 100644
index fe801ad..0000000
--- a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TestPhoenixTransactSQL.java
+++ /dev/null
@@ -1,608 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline;
-
-import org.apache.hadoop.metrics2.sink.timeline.Precision;
-import org.apache.hadoop.metrics2.sink.timeline.PrecisionLimitExceededException;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query.Condition;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query.DefaultCondition;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query.PhoenixTransactSQL;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query.SplitByMetricNamesCondition;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query.TopNCondition;
-import org.easymock.Capture;
-import org.junit.Assert;
-import org.junit.Ignore;
-import org.junit.Test;
-import java.sql.Connection;
-import java.sql.ParameterMetaData;
-import java.sql.PreparedStatement;
-import java.sql.SQLException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.List;
-
-import static org.easymock.EasyMock.createNiceMock;
-import static org.easymock.EasyMock.expect;
-import static org.easymock.EasyMock.replay;
-import static org.easymock.EasyMock.reset;
-import static org.easymock.EasyMock.verify;
-import org.easymock.EasyMock;
-
-public class TestPhoenixTransactSQL {
- @Test
- public void testConditionClause() throws Exception {
- Condition condition = new DefaultCondition(Arrays.asList(new byte[8], new byte[8]),
- new ArrayList<>(Arrays.asList("cpu_user", "mem_free")), Collections.singletonList("h1"),
- "a1", "i1", 1407959718L, 1407959918L, null, null, false);
-
- String preparedClause = condition.getConditionClause().toString();
- String expectedClause = "(UUID IN (?, ?)) AND SERVER_TIME >= ? AND SERVER_TIME < ?";
-
- Assert.assertNotNull(preparedClause);
- Assert.assertEquals(expectedClause, preparedClause);
- }
-
- @Test
- public void testSplitByMetricNamesCondition() throws Exception {
- Condition c = new DefaultCondition(Arrays.asList(new byte[8], new byte[8]),
- Arrays.asList("cpu_user", "mem_free"), Collections.singletonList("h1"),
- "a1", "i1", 1407959718L, 1407959918L, null, null, false);
-
- SplitByMetricNamesCondition condition = new SplitByMetricNamesCondition(c);
- condition.setCurrentUuid(new byte[8]);
-
- String preparedClause = condition.getConditionClause().toString();
- String expectedClause = "UUID = ? AND SERVER_TIME >= ? AND SERVER_TIME < ?";
-
- Assert.assertNotNull(preparedClause);
- Assert.assertEquals(expectedClause, preparedClause);
- }
-
- @Ignore
- @Test
- public void testLikeConditionClause() throws Exception {
- Condition condition = new DefaultCondition(
- new ArrayList<>(Arrays.asList("cpu_user", "some=%.metric")),
- Collections.singletonList("h1"), "a1", "i1", 1407959718L, 1407959918L,
- null, null, false);
-
- String preparedClause = condition.getConditionClause().toString();
- String expectedClause = "(METRIC_NAME IN (?) OR METRIC_NAME LIKE ?) AND HOSTNAME = ? AND " +
- "APP_ID = ? AND INSTANCE_ID = ? AND SERVER_TIME >= ? AND SERVER_TIME < ?";
-
- Assert.assertNotNull(preparedClause);
- Assert.assertEquals(expectedClause, preparedClause);
-
-
- condition = new DefaultCondition(
- Collections.<String>emptyList(), Collections.singletonList("h1"), "a1", "i1",
- 1407959718L, 1407959918L, null, null, false);
-
- preparedClause = condition.getConditionClause().toString();
- expectedClause = " HOSTNAME = ? AND " +
- "APP_ID = ? AND INSTANCE_ID = ? AND SERVER_TIME >= ? AND SERVER_TIME < ?";
-
- Assert.assertNotNull(preparedClause);
- Assert.assertEquals(expectedClause, preparedClause);
-
-
- condition = new DefaultCondition(
- null, Collections.singletonList("h1"), "a1", "i1",
- 1407959718L, 1407959918L, null, null, false);
-
- preparedClause = condition.getConditionClause().toString();
- expectedClause = " HOSTNAME = ? AND " +
- "APP_ID = ? AND INSTANCE_ID = ? AND SERVER_TIME >= ? AND SERVER_TIME < ?";
-
- Assert.assertNotNull(preparedClause);
- Assert.assertEquals(expectedClause, preparedClause);
-
-
- condition = new DefaultCondition(
- new ArrayList<>(Arrays.asList("some=%.metric")), Collections.singletonList("h1"), "a1", "i1",
- 1407959718L, 1407959918L, null, null, false);
-
- preparedClause = condition.getConditionClause().toString();
- expectedClause = "(METRIC_NAME LIKE ?) AND HOSTNAME = ? AND " +
- "APP_ID = ? AND INSTANCE_ID = ? AND SERVER_TIME >= ? AND SERVER_TIME < ?";
-
- Assert.assertNotNull(preparedClause);
- Assert.assertEquals(expectedClause, preparedClause);
-
-
- condition = new DefaultCondition(
- new ArrayList<>(Arrays.asList("some=%.metric1", "some=%.metric2", "some=%.metric3")),
- Collections.singletonList("h1"), "a1", "i1",
- 1407959718L, 1407959918L, null, null, false);
-
- preparedClause = condition.getConditionClause().toString();
- expectedClause = "(METRIC_NAME LIKE ? OR METRIC_NAME LIKE ? OR METRIC_NAME LIKE ?) AND HOSTNAME = ? AND " +
- "APP_ID = ? AND INSTANCE_ID = ? AND SERVER_TIME >= ? AND SERVER_TIME < ?";
-
- Assert.assertNotNull(preparedClause);
- Assert.assertEquals(expectedClause, preparedClause);
- }
-
- @Test
- public void testPrepareGetAggregatePrecisionMINUTES() throws SQLException {
- Condition condition = new DefaultCondition(
- new ArrayList<>(Arrays.asList("cpu_user", "mem_free")), Collections.singletonList("h1"),
- "a1", "i1", 1407959718L, 1407959918L, Precision.MINUTES, null, false);
- Connection connection = createNiceMock(Connection.class);
- PreparedStatement preparedStatement = createNiceMock(PreparedStatement.class);
- Capture<String> stmtCapture = new Capture<String>();
- expect(connection.prepareStatement(EasyMock.and(EasyMock.anyString(), EasyMock.capture(stmtCapture))))
- .andReturn(preparedStatement);
-
- replay(connection, preparedStatement);
- PhoenixTransactSQL.prepareGetAggregateSqlStmt(connection, condition);
- String stmt = stmtCapture.getValue();
- Assert.assertTrue(stmt.contains("FROM METRIC_AGGREGATE"));
- verify(connection, preparedStatement);
- }
-
- @Test
- public void testPrepareGetAggregateNoPrecision() throws SQLException {
- Long endTime = 1407959918L;
- Long startTime = 1407959718L;
- //SECONDS precision
- Condition condition = new DefaultCondition(
- new ArrayList<>(Arrays.asList("cpu_user", "mem_free")), Collections.singletonList("h1"),
- "a1", "i1", startTime, endTime, null, null, false);
- Connection connection = createNiceMock(Connection.class);
- PreparedStatement preparedStatement = createNiceMock(PreparedStatement.class);
- Capture<String> stmtCapture = new Capture<String>();
- expect(connection.prepareStatement(EasyMock.and(EasyMock.anyString(), EasyMock.capture(stmtCapture))))
- .andReturn(preparedStatement);
-
- replay(connection, preparedStatement);
- PhoenixTransactSQL.prepareGetAggregateSqlStmt(connection, condition);
- String stmt = stmtCapture.getValue();
- Assert.assertTrue(stmt.contains("FROM METRIC_AGGREGATE"));
- Assert.assertEquals(Precision.SECONDS, condition.getPrecision());
- verify(connection, preparedStatement);
-
- // MINUTES precision
- startTime = endTime-PhoenixTransactSQL.DAY/1000;
- condition = new DefaultCondition(
- new ArrayList<>(Arrays.asList("cpu_user", "mem_free")), Collections.singletonList("h1"),
- "a1", "i1", startTime, endTime, null, null, false);
- connection = createNiceMock(Connection.class);
- preparedStatement = createNiceMock(PreparedStatement.class);
- stmtCapture = new Capture<String>();
- expect(connection.prepareStatement(EasyMock.and(EasyMock.anyString(), EasyMock.capture(stmtCapture))))
- .andReturn(preparedStatement);
-
- replay(connection, preparedStatement);
- PhoenixTransactSQL.prepareGetAggregateSqlStmt(connection, condition);
- stmt = stmtCapture.getValue();
- Assert.assertTrue(stmt.contains("FROM METRIC_AGGREGATE"));
- Assert.assertEquals(Precision.MINUTES, condition.getPrecision());
- verify(connection, preparedStatement);
-
- // HOURS precision
- startTime = endTime-PhoenixTransactSQL.DAY*30/1000;
- condition = new DefaultCondition(
- new ArrayList<>(Arrays.asList("cpu_user", "mem_free")), Collections.singletonList("h1"),
- "a1", "i1", startTime, endTime, null, null, false);
- connection = createNiceMock(Connection.class);
- preparedStatement = createNiceMock(PreparedStatement.class);
- stmtCapture = new Capture<String>();
- expect(connection.prepareStatement(EasyMock.and(EasyMock.anyString(), EasyMock.capture(stmtCapture))))
- .andReturn(preparedStatement);
-
- replay(connection, preparedStatement);
- PhoenixTransactSQL.prepareGetAggregateSqlStmt(connection, condition);
- stmt = stmtCapture.getValue();
- Assert.assertTrue(stmt.contains("FROM METRIC_AGGREGATE_HOURLY"));
- Assert.assertEquals(Precision.HOURS, condition.getPrecision());
- verify(connection, preparedStatement);
-
- // DAYS precision
- startTime = endTime-PhoenixTransactSQL.DAY*30*2/1000;
- condition = new DefaultCondition(
- new ArrayList<>(Arrays.asList("cpu_user", "mem_free")), Collections.singletonList("h1"),
- "a1", "i1", startTime, endTime, null, null, false);
- connection = createNiceMock(Connection.class);
- preparedStatement = createNiceMock(PreparedStatement.class);
- stmtCapture = new Capture<String>();
- expect(connection.prepareStatement(EasyMock.and(EasyMock.anyString(), EasyMock.capture(stmtCapture))))
- .andReturn(preparedStatement);
-
- replay(connection, preparedStatement);
- PhoenixTransactSQL.prepareGetAggregateSqlStmt(connection, condition);
- stmt = stmtCapture.getValue();
- Assert.assertTrue(stmt.contains("FROM METRIC_AGGREGATE_DAILY"));
- Assert.assertEquals(Precision.DAYS, condition.getPrecision());
- verify(connection, preparedStatement);
- }
-
- @Test
- public void testPrepareGetAggregatePrecisionHours() throws SQLException {
- Condition condition = new DefaultCondition(
- new ArrayList<>(Arrays.asList("cpu_user", "mem_free")), Collections.singletonList("h1"),
- "a1", "i1", 1407959718L, 1407959918L, Precision.HOURS, null, false);
- Connection connection = createNiceMock(Connection.class);
- PreparedStatement preparedStatement = createNiceMock(PreparedStatement.class);
- Capture<String> stmtCapture = new Capture<String>();
- expect(connection.prepareStatement(EasyMock.and(EasyMock.anyString(), EasyMock.capture(stmtCapture))))
- .andReturn(preparedStatement);
-
- replay(connection, preparedStatement);
- PhoenixTransactSQL.prepareGetAggregateSqlStmt(connection, condition);
- String stmt = stmtCapture.getValue();
- Assert.assertTrue(stmt.contains("FROM METRIC_AGGREGATE_HOURLY"));
- verify(connection, preparedStatement);
- }
-
- @Test
- public void testPrepareGetMetricsPrecisionMinutes() throws SQLException {
- Condition condition = new DefaultCondition(
- new ArrayList<>(Arrays.asList("cpu_user", "mem_free")), Collections.singletonList("h1"),
- "a1", "i1", 1407959718L, 1407959918L, Precision.MINUTES, null, false);
- Connection connection = createNiceMock(Connection.class);
- PreparedStatement preparedStatement = createNiceMock(PreparedStatement.class);
- Capture<String> stmtCapture = new Capture<String>();
- expect(connection.prepareStatement(EasyMock.and(EasyMock.anyString(), EasyMock.capture(stmtCapture))))
- .andReturn(preparedStatement);
-
- replay(connection, preparedStatement);
- PhoenixTransactSQL.prepareGetMetricsSqlStmt(connection, condition);
- String stmt = stmtCapture.getValue();
- Assert.assertTrue(stmt.contains("FROM METRIC_RECORD_MINUTE"));
- verify(connection, preparedStatement);
- }
-
- @Test
- public void testPrepareGetMetricsNoPrecision() throws SQLException {
- Long endTime = 1407959918L;
- Long startTime = endTime - 200;
- // SECONDS precision
- Condition condition = new DefaultCondition(
- new ArrayList<>(Arrays.asList("cpu_user", "mem_free")), Collections.singletonList("h1"),
- "a1", "i1", startTime, endTime, null, null, false);
- Connection connection = createNiceMock(Connection.class);
- PreparedStatement preparedStatement = createNiceMock(PreparedStatement.class);
- Capture<String> stmtCapture = new Capture<String>();
- expect(connection.prepareStatement(EasyMock.and(EasyMock.anyString(), EasyMock.capture(stmtCapture))))
- .andReturn(preparedStatement);
-
- replay(connection, preparedStatement);
- PhoenixTransactSQL.prepareGetMetricsSqlStmt(connection, condition);
- String stmt = stmtCapture.getValue();
- Assert.assertTrue(stmt.contains("FROM METRIC_RECORD"));
- Assert.assertEquals(Precision.SECONDS, condition.getPrecision());
- verify(connection, preparedStatement);
- reset(connection, preparedStatement);
-
- // SECONDS precision
- startTime = endTime-PhoenixTransactSQL.HOUR*2/1000;
- condition = new DefaultCondition(
- new ArrayList<>(Arrays.asList("cpu_user", "mem_free")), Collections.singletonList("h1"),
- "a1", "i1", startTime, endTime, null, null, false);
- connection = createNiceMock(Connection.class);
- preparedStatement = createNiceMock(PreparedStatement.class);
- stmtCapture = new Capture<String>();
- expect(connection.prepareStatement(EasyMock.and(EasyMock.anyString(), EasyMock.capture(stmtCapture))))
- .andReturn(preparedStatement);
- replay(connection, preparedStatement);
- PhoenixTransactSQL.prepareGetMetricsSqlStmt(connection, condition);
- stmt = stmtCapture.getValue();
- Assert.assertTrue(stmt.contains("FROM METRIC_RECORD"));
- Assert.assertEquals(Precision.SECONDS, condition.getPrecision());
- verify(connection, preparedStatement);
-
- // MINUTES precision
- startTime = endTime-PhoenixTransactSQL.DAY/1000;
- condition = new DefaultCondition(
- new ArrayList<>(Arrays.asList("cpu_user", "mem_free")), Collections.singletonList("h1"),
- "a1", "i1", startTime, endTime, null, null, false);
- connection = createNiceMock(Connection.class);
- preparedStatement = createNiceMock(PreparedStatement.class);
- stmtCapture = new Capture<String>();
- expect(connection.prepareStatement(EasyMock.and(EasyMock.anyString(), EasyMock.capture(stmtCapture))))
- .andReturn(preparedStatement);
- replay(connection, preparedStatement);
- PhoenixTransactSQL.prepareGetMetricsSqlStmt(connection, condition);
- stmt = stmtCapture.getValue();
- Assert.assertTrue(stmt.contains("FROM METRIC_RECORD_MINUTE"));
- Assert.assertEquals(Precision.MINUTES, condition.getPrecision());
- verify(connection, preparedStatement);
-
- // HOURS precision
- startTime = endTime-PhoenixTransactSQL.DAY*30/1000;
- condition = new DefaultCondition(
- new ArrayList<>(Arrays.asList("cpu_user", "mem_free")), Collections.singletonList("h1"),
- "a1", "i1", startTime, endTime, null, null, false);
- connection = createNiceMock(Connection.class);
- preparedStatement = createNiceMock(PreparedStatement.class);
- stmtCapture = new Capture<String>();
- expect(connection.prepareStatement(EasyMock.and(EasyMock.anyString(), EasyMock.capture(stmtCapture))))
- .andReturn(preparedStatement);
- replay(connection, preparedStatement);
- PhoenixTransactSQL.prepareGetMetricsSqlStmt(connection, condition);
- stmt = stmtCapture.getValue();
- Assert.assertTrue(stmt.contains("FROM METRIC_RECORD_HOURLY"));
- Assert.assertEquals(Precision.HOURS, condition.getPrecision());
- verify(connection, preparedStatement);
-
- // DAYS precision
- startTime = endTime-PhoenixTransactSQL.DAY*30*2/1000;
- condition = new DefaultCondition(
- new ArrayList<>(Arrays.asList("cpu_user", "mem_free")), Collections.singletonList("h1"),
- "a1", "i1", startTime, endTime, null, null, false);
- connection = createNiceMock(Connection.class);
- preparedStatement = createNiceMock(PreparedStatement.class);
- stmtCapture = new Capture<String>();
- expect(connection.prepareStatement(EasyMock.and(EasyMock.anyString(), EasyMock.capture(stmtCapture))))
- .andReturn(preparedStatement);
- replay(connection, preparedStatement);
- PhoenixTransactSQL.prepareGetMetricsSqlStmt(connection, condition);
- stmt = stmtCapture.getValue();
- Assert.assertTrue(stmt.contains("FROM METRIC_RECORD_DAILY"));
- Assert.assertEquals(Precision.DAYS, condition.getPrecision());
- verify(connection, preparedStatement);
-
- }
-
- @Test
- public void testPrepareGetLatestMetricSqlStmtMultipleHostNames() throws SQLException {
- Condition condition = new DefaultCondition(Arrays.asList(new byte[16], new byte[16], new byte[16], new byte[16]),
- new ArrayList<>(Arrays.asList("cpu_user", "mem_free")), Arrays.asList("h1", "h2"),
- "a1", "i1", null, null, null, null, false);
- Connection connection = createNiceMock(Connection.class);
- PreparedStatement preparedStatement = createNiceMock(PreparedStatement.class);
- ParameterMetaData parameterMetaData = createNiceMock(ParameterMetaData.class);
- Capture<String> stmtCapture = new Capture<String>();
- expect(connection.prepareStatement(EasyMock.and(EasyMock.anyString(), EasyMock.capture(stmtCapture))))
- .andReturn(preparedStatement);
- expect(preparedStatement.getParameterMetaData())
- .andReturn(parameterMetaData).once();
- // 6 = 1 instance_id + 1 appd_id + 2 hostnames + 2 metric names
- expect(parameterMetaData.getParameterCount())
- .andReturn(4).once();
-
- replay(connection, preparedStatement, parameterMetaData);
- PhoenixTransactSQL.prepareGetLatestMetricSqlStmt(connection, condition);
- String stmt = stmtCapture.getValue();
- Assert.assertTrue(stmt.contains("FROM METRIC_RECORD"));
- Assert.assertTrue(stmt.contains("JOIN"));
- verify(connection, preparedStatement, parameterMetaData);
- }
-
- @Test
- public void testPrepareGetLatestMetricSqlStmtSortMergeJoinAlgorithm()
- throws SQLException {
- Condition condition = new DefaultCondition(Arrays.asList(new byte[16], new byte[16]),
- new ArrayList<>(Arrays.asList("cpu_user", "mem_free")), Arrays.asList("h1"),
- "a1", "i1", null, null, null, null, false);
- Connection connection = createNiceMock(Connection.class);
- PreparedStatement preparedStatement = createNiceMock(PreparedStatement.class);
- ParameterMetaData parameterMetaData = createNiceMock(ParameterMetaData.class);
- Capture<String> stmtCapture = new Capture<String>();
- expect(connection.prepareStatement(EasyMock.and(EasyMock.anyString(), EasyMock.capture(stmtCapture))))
- .andReturn(preparedStatement);
- expect(preparedStatement.getParameterMetaData())
- .andReturn(parameterMetaData).anyTimes();
- expect(parameterMetaData.getParameterCount())
- .andReturn(2).anyTimes();
-
- replay(connection, preparedStatement, parameterMetaData);
- PhoenixTransactSQL.setSortMergeJoinEnabled(true);
- PhoenixTransactSQL.prepareGetLatestMetricSqlStmt(connection, condition);
- String stmt = stmtCapture.getValue();
- Assert.assertTrue(stmt.contains("/*+ USE_SORT_MERGE_JOIN NO_CACHE */"));
- }
-
- @Test
- public void testPrepareGetMetricsPrecisionHours() throws SQLException {
- Condition condition = new DefaultCondition(
- new ArrayList<>(Arrays.asList("cpu_user", "mem_free")), Collections.singletonList("h1"),
- "a1", "i1", 1407959718L, 1407959918L, Precision.HOURS, null, false);
- Connection connection = createNiceMock(Connection.class);
- PreparedStatement preparedStatement = createNiceMock(PreparedStatement.class);
- Capture<String> stmtCapture = new Capture<String>();
- expect(connection.prepareStatement(EasyMock.and(EasyMock.anyString(), EasyMock.capture(stmtCapture))))
- .andReturn(preparedStatement);
-
- replay(connection, preparedStatement);
- PhoenixTransactSQL.prepareGetMetricsSqlStmt(connection, condition);
- String stmt = stmtCapture.getValue();
- Assert.assertTrue(stmt.contains("FROM METRIC_RECORD_HOURLY"));
- verify(connection, preparedStatement);
- }
-
- @Test
- public void testResultSetLimitCheck() throws SQLException {
-
- List<String> metrics = new ArrayList<String>();
- List<String> hosts = new ArrayList<String>();
- int numMetrics = 0;
- int numHosts = 0;
- int limit = PhoenixHBaseAccessor.RESULTSET_LIMIT;
-
- // 22 Metrics x 2 Hosts x 1 hour with requested SECONDS precision = 15840 points. Should be OK!
- numMetrics = 22;
- numHosts = 2;
- for (int i = 0; i < numMetrics; i++) {
- metrics.add("TestMetric"+i);
- }
- for (int i = 0; i < numHosts; i++) {
- hosts.add("TestHost"+i);
- }
-
- Condition condition = new DefaultCondition(
- metrics, hosts,
- "a1", "i1", 1407950000L, 1407953600L, Precision.SECONDS, null, false);
- Connection connection = createNiceMock(Connection.class);
- PreparedStatement preparedStatement = createNiceMock(PreparedStatement.class);
- Capture<String> stmtCapture = new Capture<String>();
- expect(connection.prepareStatement(EasyMock.and(EasyMock.anyString(), EasyMock.capture(stmtCapture))))
- .andReturn(preparedStatement);
-
- replay(connection, preparedStatement);
- PhoenixTransactSQL.prepareGetMetricsSqlStmt(connection, condition);
- String stmt = stmtCapture.getValue();
- Assert.assertTrue(stmt.contains("FROM METRIC_RECORD"));
- verify(connection, preparedStatement);
-
- //Check without passing precision. Should be OK!
- condition = new DefaultCondition(
- metrics, hosts,
- "a1", "i1", 1407950000L, 1407953600L, null, null, false);
- connection = createNiceMock(Connection.class);
- preparedStatement = createNiceMock(PreparedStatement.class);
- stmtCapture = new Capture<String>();
- expect(connection.prepareStatement(EasyMock.and(EasyMock.anyString(), EasyMock.capture(stmtCapture))))
- .andReturn(preparedStatement);
-
- replay(connection, preparedStatement);
- PhoenixTransactSQL.prepareGetMetricsSqlStmt(connection, condition);
- stmt = stmtCapture.getValue();
- Assert.assertTrue(stmt.contains("FROM METRIC_RECORD"));
- verify(connection, preparedStatement);
-
- //Check with more hosts and lesser metrics for 1 day data = 11520 points Should be OK!
- metrics.clear();
- hosts.clear();
- numMetrics = 2;
- numHosts = 20;
- for (int i = 0; i < numMetrics; i++) {
- metrics.add("TestMetric"+i);
- }
- for (int i = 0; i < numHosts; i++) {
- hosts.add("TestHost"+i);
- }
- condition = new DefaultCondition(
- metrics, hosts,
- "a1", "i1", 1407867200L, 1407953600L, null, null, false);
- connection = createNiceMock(Connection.class);
- preparedStatement = createNiceMock(PreparedStatement.class);
- stmtCapture = new Capture<String>();
- expect(connection.prepareStatement(EasyMock.and(EasyMock.anyString(), EasyMock.capture(stmtCapture))))
- .andReturn(preparedStatement);
-
- replay(connection, preparedStatement);
- PhoenixTransactSQL.prepareGetMetricsSqlStmt(connection, condition);
- stmt = stmtCapture.getValue();
- Assert.assertTrue(stmt.contains("FROM METRIC_RECORD_MINUTE"));
- verify(connection, preparedStatement);
-
- //Check with 20 metrics, NO hosts and 1 day data = 5760 points. Should be OK!
- metrics.clear();
- hosts.clear();
- numMetrics = 20;
- for (int i = 0; i < numMetrics; i++) {
- metrics.add("TestMetric"+i);
- }
- condition = new DefaultCondition(
- metrics, hosts,
- "a1", "i1", 1407867200L, 1407953600L, null, null, false);
- connection = createNiceMock(Connection.class);
- preparedStatement = createNiceMock(PreparedStatement.class);
- stmtCapture = new Capture<String>();
- expect(connection.prepareStatement(EasyMock.and(EasyMock.anyString(), EasyMock.capture(stmtCapture))))
- .andReturn(preparedStatement);
-
- replay(connection, preparedStatement);
- PhoenixTransactSQL.prepareGetAggregateSqlStmt(connection, condition);
- stmt = stmtCapture.getValue();
- Assert.assertTrue(stmt.contains("FROM METRIC_AGGREGATE_MINUTE"));
- verify(connection, preparedStatement);
-
- //Check with 5 hosts and 10 metrics for 1 hour data = 18000 points. Should throw out Exception!
- metrics.clear();
- hosts.clear();
- numMetrics = 10;
- numHosts = 5;
- for (int i = 0; i < numMetrics; i++) {
- metrics.add("TestMetric"+i);
- }
- for (int i = 0; i < numHosts; i++) {
- hosts.add("TestHost"+i);
- }
- condition = new DefaultCondition(
- metrics, hosts,
- "a1", "i1", 1407950000L, 1407953600L, null, null, false);
- boolean exceptionThrown = false;
- boolean requestedSizeFoundInMessage = false;
-
- try {
- PhoenixTransactSQL.prepareGetMetricsSqlStmt(connection, condition);
- } catch (PrecisionLimitExceededException pe) {
- exceptionThrown = true;
- String message = pe.getMessage();
- if (message !=null && message.contains("18000")) {
- requestedSizeFoundInMessage = true;
- }
- }
- Assert.assertTrue(exceptionThrown);
- Assert.assertTrue(requestedSizeFoundInMessage);
- }
-
- @Test
- public void testTopNHostsConditionClause() throws Exception {
- List<String> hosts = Arrays.asList("h1", "h2");
- List<byte[]> uuids = Arrays.asList(new byte[16], new byte[16]);
-
- Condition condition = new TopNCondition(uuids, new ArrayList<>(Collections.singletonList("cpu_user")), hosts,
- "a1", "i1", 1407959718L, 1407959918L, null, null, false, 2, null, false);
-
- String conditionClause = condition.getConditionClause().toString();
- String expectedClause = " UUID IN (" +
- "SELECT UUID FROM METRIC_RECORD WHERE " +
- "(UUID LIKE ? OR UUID LIKE ?) AND " +
- "SERVER_TIME >= ? AND SERVER_TIME < ? " +
- "GROUP BY UUID ORDER BY MAX(METRIC_MAX) DESC LIMIT 2) AND SERVER_TIME >= ? AND SERVER_TIME < ?";
-
- Assert.assertEquals(expectedClause, conditionClause);
- }
-
- @Test
- public void testTopNMetricsConditionClause() throws Exception {
- List<String> metricNames = new ArrayList<>(Arrays.asList("m1", "m2", "m3"));
- List<byte[]> uuids = Arrays.asList(new byte[16], new byte[16], new byte[16]);
-
- Condition condition = new TopNCondition(uuids, metricNames, Collections.singletonList("h1"),
- "a1", "i1", 1407959718L, 1407959918L, null, null, false, 2, null, false);
-
- String conditionClause = condition.getConditionClause().toString();
- String expectedClause = " UUID IN (" +
- "SELECT UUID FROM METRIC_RECORD WHERE " +
- "(UUID LIKE ? OR UUID LIKE ? OR UUID LIKE ?) AND " +
- "SERVER_TIME >= ? AND SERVER_TIME < ? " +
- "GROUP BY UUID ORDER BY MAX(METRIC_MAX) DESC LIMIT 2) AND SERVER_TIME >= ? AND SERVER_TIME < ?";
-
- Assert.assertEquals(expectedClause, conditionClause);
- }
-
- @Test
- public void testTopNMetricsIllegalConditionClause() throws Exception {
- List<String> metricNames = new ArrayList<>(Arrays.asList("m1", "m2"));
-
- List<String> hosts = Arrays.asList("h1", "h2");
- List<byte[]> uuids = Arrays.asList(new byte[16], new byte[16], new byte[16], new byte[16]);
-
- Condition condition = new TopNCondition(uuids, metricNames, hosts,
- "a1", "i1", 1407959718L, 1407959918L, null, null, false, 2, null, false);
-
- Assert.assertEquals(condition.getConditionClause(), null);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TestTimelineMetricStore.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TestTimelineMetricStore.java b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TestTimelineMetricStore.java
deleted file mode 100644
index 9ff2e07..0000000
--- a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TestTimelineMetricStore.java
+++ /dev/null
@@ -1,124 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline;
-
-import java.io.IOException;
-import java.sql.SQLException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.TreeMap;
-
-import org.apache.hadoop.metrics2.sink.timeline.AggregationResult;
-import org.apache.hadoop.metrics2.sink.timeline.ContainerMetric;
-import org.apache.hadoop.metrics2.sink.timeline.Precision;
-import org.apache.hadoop.metrics2.sink.timeline.TimelineMetric;
-import org.apache.hadoop.metrics2.sink.timeline.TimelineMetricMetadata;
-import org.apache.hadoop.metrics2.sink.timeline.TimelineMetrics;
-import org.apache.hadoop.metrics2.sink.timeline.TopNConfig;
-import org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.discovery.TimelineMetricMetadataKey;
-
-public class TestTimelineMetricStore implements TimelineMetricStore {
- @Override
- public TimelineMetrics getTimelineMetrics(List<String> metricNames,
- List<String> hostnames, String applicationId, String instanceId, Long startTime,
- Long endTime, Precision precision, Integer limit, boolean groupedByHost,
- TopNConfig topNConfig, String seriesAggregateFunction) throws SQLException,
- IOException {
- TimelineMetrics timelineMetrics = new TimelineMetrics();
- List<TimelineMetric> metricList = new ArrayList<TimelineMetric>();
- timelineMetrics.setMetrics(metricList);
- TimelineMetric metric1 = new TimelineMetric();
- TimelineMetric metric2 = new TimelineMetric();
- metricList.add(metric1);
- metricList.add(metric2);
- metric1.setMetricName("cpu_user");
- metric1.setAppId("1");
- metric1.setInstanceId(null);
- metric1.setHostName("c6401");
- metric1.setStartTime(1407949812L);
- metric1.setMetricValues(new TreeMap<Long, Double>() {{
- put(1407949812L, 1.0d);
- put(1407949912L, 1.8d);
- put(1407950002L, 0.7d);
- }});
-
- metric2.setMetricName("mem_free");
- metric2.setAppId("2");
- metric2.setInstanceId("3");
- metric2.setHostName("c6401");
- metric2.setStartTime(1407949812L);
- metric2.setMetricValues(new TreeMap<Long, Double>() {{
- put(1407949812L, 2.5d);
- put(1407949912L, 3.0d);
- put(1407950002L, 0.9d);
- }});
-
- return timelineMetrics;
- }
-
- @Override
- public TimelinePutResponse putMetrics(TimelineMetrics metrics)
- throws SQLException, IOException {
-
- return new TimelinePutResponse();
- }
-
- @Override
- public TimelinePutResponse putContainerMetrics(List<ContainerMetric> metrics)
- throws SQLException, IOException {
- return new TimelinePutResponse();
- }
-
- @Override
- public Map<String, List<TimelineMetricMetadata>> getTimelineMetricMetadata(String appId, String metricPattern,
- boolean includeBlacklistedMetrics) throws SQLException, IOException {
- return null;
- }
-
- @Override
- public TimelinePutResponse putHostAggregatedMetrics(AggregationResult aggregationResult) throws SQLException, IOException {
- return null;
- }
-
- @Override
- public Map<String, Set<String>> getHostAppsMetadata() throws SQLException, IOException {
- return Collections.emptyMap();
- }
-
- @Override
- public Map<String, Map<String,Set<String>>> getInstanceHostsMetadata(String instanceId, String appId) throws SQLException, IOException {
- return Collections.emptyMap();
- }
-
- @Override
- public List<String> getLiveInstances() {
- return Collections.emptyList();
- }
-
- @Override
- public byte[] getUuid(String metricName, String appId, String instanceId, String hostname) throws SQLException, IOException {
- return null;
- }
-
-}
-
-
[28/28] ambari git commit: AMBARI-22740 : Fix integration test for
HBase in branch-3.0-ams due to UUID changes. (avijayan)
Posted by av...@apache.org.
AMBARI-22740 : Fix integration test for HBase in branch-3.0-ams due to UUID changes. (avijayan)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/42112e28
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/42112e28
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/42112e28
Branch: refs/heads/branch-3.0-ams
Commit: 42112e28d12f7cd04e4a2d4e19204ea32d2d3ebf
Parents: a4293b7
Author: Aravindan Vijayan <av...@hortonworks.com>
Authored: Mon Jan 8 10:19:15 2018 -0800
Committer: Aravindan Vijayan <av...@hortonworks.com>
Committed: Mon Jan 8 10:19:15 2018 -0800
----------------------------------------------------------------------
.../metrics2/sink/timeline/Precision.java | 1 -
.../conf/unix/ambari-metrics-collector | 2 +-
.../ambari-metrics-timelineservice/pom.xml | 2 +-
.../ambari/metrics/AMSApplicationServer.java | 141 ++
.../metrics/core/loadsimulator/LoadRunner.java | 154 ++
.../loadsimulator/MetricsLoadSimulator.java | 138 ++
.../core/loadsimulator/MetricsSenderWorker.java | 60 +
.../metrics/core/loadsimulator/data/AppID.java | 45 +
.../core/loadsimulator/data/AppMetrics.java | 47 +
.../loadsimulator/data/ApplicationInstance.java | 58 +
.../data/HostMetricsGenerator.java | 61 +
.../metrics/core/loadsimulator/data/Metric.java | 71 +
.../data/MetricsGeneratorConfigurer.java | 93 +
.../core/loadsimulator/net/MetricsSender.java | 31 +
.../loadsimulator/net/RestMetricsSender.java | 92 +
.../loadsimulator/net/StdOutMetricsSender.java | 56 +
.../core/loadsimulator/net/UrlService.java | 100 +
.../metrics/core/loadsimulator/util/Json.java | 62 +
.../util/RandomMetricsProvider.java | 39 +
.../loadsimulator/util/TimeStampProvider.java | 51 +
.../timeline/HBaseTimelineMetricsService.java | 592 ++++++
.../timeline/MetricsCacheCommitterThread.java | 38 +
.../MetricsSystemInitializationException.java | 41 +
.../core/timeline/PhoenixHBaseAccessor.java | 1844 +++++++++++++++++
.../timeline/TimelineMetricConfiguration.java | 712 +++++++
.../TimelineMetricDistributedCache.java | 32 +
.../core/timeline/TimelineMetricStore.java | 110 +
.../timeline/TimelineMetricStoreWatcher.java | 126 ++
.../timeline/TimelineMetricsAggregatorSink.java | 60 +
.../core/timeline/TimelineMetricsFilter.java | 165 ++
.../timeline/TimelineMetricsIgniteCache.java | 288 +++
.../aggregators/AbstractTimelineAggregator.java | 494 +++++
.../timeline/aggregators/AggregatorUtils.java | 254 +++
.../timeline/aggregators/CustomDownSampler.java | 44 +
.../timeline/aggregators/DownSamplerUtils.java | 117 ++
.../core/timeline/aggregators/Function.java | 214 ++
.../aggregators/TimelineClusterMetric.java | 97 +
.../aggregators/TimelineMetricAggregator.java | 59 +
.../TimelineMetricAggregatorFactory.java | 528 +++++
.../TimelineMetricAppAggregator.java | 187 ++
.../TimelineMetricClusterAggregator.java | 150 ++
.../TimelineMetricClusterAggregatorSecond.java | 263 +++
...cClusterAggregatorSecondWithCacheSource.java | 104 +
.../TimelineMetricFilteringHostAggregator.java | 93 +
.../TimelineMetricHostAggregator.java | 120 ++
.../aggregators/TimelineMetricReadHelper.java | 170 ++
.../timeline/aggregators/TopNDownSampler.java | 112 ++
.../v2/TimelineMetricClusterAggregator.java | 93 +
.../TimelineMetricFilteringHostAggregator.java | 119 ++
.../v2/TimelineMetricHostAggregator.java | 75 +
.../availability/AggregationTaskRunner.java | 141 ++
.../availability/CheckpointManager.java | 95 +
.../MetricCollectorHAController.java | 307 +++
.../OnlineOfflineStateModelFactory.java | 69 +
.../discovery/TimelineMetricHostMetadata.java | 60 +
.../discovery/TimelineMetricMetadataKey.java | 80 +
.../TimelineMetricMetadataManager.java | 606 ++++++
.../discovery/TimelineMetricMetadataSync.java | 227 +++
...tTimelineMetricsSeriesAggregateFunction.java | 98 +
.../function/SeriesAggregateFunction.java | 42 +
.../TimelineMetricsSeriesAggregateFunction.java | 25 +
...neMetricsSeriesAggregateFunctionFactory.java | 41 +
...melineMetricsSeriesAvgAggregateFunction.java | 39 +
...melineMetricsSeriesMaxAggregateFunction.java | 41 +
...melineMetricsSeriesMinAggregateFunction.java | 41 +
...melineMetricsSeriesSumAggregateFunction.java | 39 +
.../metrics/core/timeline/query/Condition.java | 51 +
.../core/timeline/query/ConditionBuilder.java | 143 ++
.../core/timeline/query/ConnectionProvider.java | 29 +
.../core/timeline/query/DefaultCondition.java | 421 ++++
.../query/DefaultPhoenixDataSource.java | 90 +
.../core/timeline/query/EmptyCondition.java | 169 ++
.../query/PhoenixConnectionProvider.java | 31 +
.../core/timeline/query/PhoenixTransactSQL.java | 803 ++++++++
.../query/SplitByMetricNamesCondition.java | 189 ++
.../core/timeline/query/TopNCondition.java | 162 ++
.../timeline/sink/DefaultFSSinkProvider.java | 151 ++
.../core/timeline/sink/ExternalMetricsSink.java | 48 +
.../timeline/sink/ExternalSinkProvider.java | 35 +
.../core/timeline/sink/HttpSinkProvider.java | 231 +++
.../core/timeline/sink/KafkaSinkProvider.java | 109 +
.../DefaultInternalMetricsSourceProvider.java | 42 +
.../timeline/source/InternalMetricsSource.java | 30 +
.../timeline/source/InternalSourceProvider.java | 39 +
.../core/timeline/source/RawMetricsSource.java | 85 +
.../source/cache/InternalMetricCacheKey.java | 109 +
.../source/cache/InternalMetricCacheValue.java | 37 +
.../source/cache/InternalMetricsCache.java | 229 +++
.../cache/InternalMetricsCacheProvider.java | 48 +
.../cache/InternalMetricsCacheSizeOfEngine.java | 52 +
.../timeline/uuid/HashBasedUuidGenStrategy.java | 209 ++
.../timeline/uuid/MetricUuidGenStrategy.java | 48 +
.../timeline/uuid/RandomUuidGenStrategy.java | 53 +
.../records/ApplicationAttemptFinishData.java | 95 +
.../records/ApplicationAttemptHistoryData.java | 171 ++
.../records/ApplicationAttemptStartData.java | 82 +
.../metrics/records/ApplicationFinishData.java | 94 +
.../metrics/records/ApplicationHistoryData.java | 213 ++
.../metrics/records/ApplicationStartData.java | 106 +
.../metrics/records/ContainerFinishData.java | 90 +
.../metrics/records/ContainerHistoryData.java | 182 ++
.../metrics/records/ContainerStartData.java | 92 +
.../pb/ApplicationAttemptFinishDataPBImpl.java | 239 +++
.../pb/ApplicationAttemptStartDataPBImpl.java | 208 ++
.../impl/pb/ApplicationFinishDataPBImpl.java | 226 +++
.../impl/pb/ApplicationStartDataPBImpl.java | 229 +++
.../impl/pb/ContainerFinishDataPBImpl.java | 204 ++
.../impl/pb/ContainerStartDataPBImpl.java | 258 +++
.../metrics/timeline/GenericObjectMapper.java | 135 ++
.../ambari/metrics/timeline/NameValuePair.java | 59 +
.../ambari/metrics/timeline/TimelineReader.java | 155 ++
.../ambari/metrics/timeline/TimelineStore.java | 29 +
.../ambari/metrics/timeline/TimelineWriter.java | 46 +
.../ambari/metrics/webapp/AMSController.java | 37 +
.../apache/ambari/metrics/webapp/AMSWebApp.java | 42 +
.../metrics/webapp/JAXBContextResolver.java | 64 +
.../metrics/webapp/TimelineWebServices.java | 513 +++++
.../AMSApplicationServer.java | 141 --
.../metrics/loadsimulator/LoadRunner.java | 155 --
.../loadsimulator/MetricsLoadSimulator.java | 139 --
.../loadsimulator/MetricsSenderWorker.java | 60 -
.../metrics/loadsimulator/data/AppID.java | 46 -
.../metrics/loadsimulator/data/AppMetrics.java | 48 -
.../loadsimulator/data/ApplicationInstance.java | 59 -
.../data/HostMetricsGenerator.java | 61 -
.../metrics/loadsimulator/data/Metric.java | 71 -
.../data/MetricsGeneratorConfigurer.java | 93 -
.../loadsimulator/net/MetricsSender.java | 31 -
.../loadsimulator/net/RestMetricsSender.java | 92 -
.../loadsimulator/net/StdOutMetricsSender.java | 56 -
.../metrics/loadsimulator/net/UrlService.java | 101 -
.../metrics/loadsimulator/util/Json.java | 63 -
.../util/RandomMetricsProvider.java | 40 -
.../loadsimulator/util/TimeStampProvider.java | 52 -
.../timeline/HBaseTimelineMetricsService.java | 594 ------
.../timeline/MetricsCacheCommitterThread.java | 38 -
.../MetricsSystemInitializationException.java | 42 -
.../metrics/timeline/PhoenixHBaseAccessor.java | 1904 ------------------
.../timeline/TimelineMetricConfiguration.java | 713 -------
.../TimelineMetricDistributedCache.java | 32 -
.../metrics/timeline/TimelineMetricStore.java | 110 -
.../timeline/TimelineMetricStoreWatcher.java | 126 --
.../timeline/TimelineMetricsAggregatorSink.java | 60 -
.../metrics/timeline/TimelineMetricsFilter.java | 170 --
.../timeline/TimelineMetricsIgniteCache.java | 296 ---
.../aggregators/AbstractTimelineAggregator.java | 495 -----
.../timeline/aggregators/AggregatorUtils.java | 254 ---
.../timeline/aggregators/CustomDownSampler.java | 44 -
.../timeline/aggregators/DownSamplerUtils.java | 117 --
.../metrics/timeline/aggregators/Function.java | 212 --
.../aggregators/TimelineClusterMetric.java | 97 -
.../aggregators/TimelineMetricAggregator.java | 59 -
.../TimelineMetricAggregatorFactory.java | 528 -----
.../TimelineMetricAppAggregator.java | 189 --
.../TimelineMetricClusterAggregator.java | 145 --
.../TimelineMetricClusterAggregatorSecond.java | 263 ---
...cClusterAggregatorSecondWithCacheSource.java | 104 -
.../TimelineMetricFilteringHostAggregator.java | 94 -
.../TimelineMetricHostAggregator.java | 121 --
.../aggregators/TimelineMetricReadHelper.java | 170 --
.../timeline/aggregators/TopNDownSampler.java | 112 --
.../v2/TimelineMetricClusterAggregator.java | 93 -
.../TimelineMetricFilteringHostAggregator.java | 119 --
.../v2/TimelineMetricHostAggregator.java | 76 -
.../availability/AggregationTaskRunner.java | 145 --
.../availability/CheckpointManager.java | 98 -
.../MetricCollectorHAController.java | 307 ---
.../OnlineOfflineStateModelFactory.java | 69 -
.../discovery/TimelineMetricHostMetadata.java | 60 -
.../discovery/TimelineMetricMetadataKey.java | 80 -
.../TimelineMetricMetadataManager.java | 612 ------
.../discovery/TimelineMetricMetadataSync.java | 227 ---
...tTimelineMetricsSeriesAggregateFunction.java | 98 -
.../function/SeriesAggregateFunction.java | 42 -
.../TimelineMetricsSeriesAggregateFunction.java | 25 -
...neMetricsSeriesAggregateFunctionFactory.java | 41 -
...melineMetricsSeriesAvgAggregateFunction.java | 39 -
...melineMetricsSeriesMaxAggregateFunction.java | 41 -
...melineMetricsSeriesMinAggregateFunction.java | 41 -
...melineMetricsSeriesSumAggregateFunction.java | 39 -
.../metrics/timeline/query/Condition.java | 51 -
.../timeline/query/ConditionBuilder.java | 143 --
.../timeline/query/ConnectionProvider.java | 29 -
.../timeline/query/DefaultCondition.java | 421 ----
.../query/DefaultPhoenixDataSource.java | 90 -
.../metrics/timeline/query/EmptyCondition.java | 169 --
.../query/PhoenixConnectionProvider.java | 31 -
.../timeline/query/PhoenixTransactSQL.java | 804 --------
.../query/SplitByMetricNamesCondition.java | 189 --
.../metrics/timeline/query/TopNCondition.java | 162 --
.../timeline/sink/DefaultFSSinkProvider.java | 153 --
.../timeline/sink/ExternalMetricsSink.java | 48 -
.../timeline/sink/ExternalSinkProvider.java | 35 -
.../metrics/timeline/sink/HttpSinkProvider.java | 231 ---
.../timeline/sink/KafkaSinkProvider.java | 118 --
.../DefaultInternalMetricsSourceProvider.java | 42 -
.../timeline/source/InternalMetricsSource.java | 30 -
.../timeline/source/InternalSourceProvider.java | 39 -
.../timeline/source/RawMetricsSource.java | 85 -
.../source/cache/InternalMetricCacheKey.java | 109 -
.../source/cache/InternalMetricCacheValue.java | 37 -
.../source/cache/InternalMetricsCache.java | 229 ---
.../cache/InternalMetricsCacheProvider.java | 48 -
.../cache/InternalMetricsCacheSizeOfEngine.java | 52 -
.../timeline/uuid/HashBasedUuidGenStrategy.java | 206 --
.../timeline/uuid/MetricUuidGenStrategy.java | 48 -
.../timeline/uuid/RandomUuidGenStrategy.java | 53 -
.../records/ApplicationAttemptFinishData.java | 95 -
.../records/ApplicationAttemptHistoryData.java | 171 --
.../records/ApplicationAttemptStartData.java | 82 -
.../records/ApplicationFinishData.java | 94 -
.../records/ApplicationHistoryData.java | 213 --
.../records/ApplicationStartData.java | 106 -
.../records/ContainerFinishData.java | 90 -
.../records/ContainerHistoryData.java | 182 --
.../records/ContainerStartData.java | 92 -
.../pb/ApplicationAttemptFinishDataPBImpl.java | 239 ---
.../pb/ApplicationAttemptStartDataPBImpl.java | 208 --
.../impl/pb/ApplicationFinishDataPBImpl.java | 226 ---
.../impl/pb/ApplicationStartDataPBImpl.java | 229 ---
.../impl/pb/ContainerFinishDataPBImpl.java | 204 --
.../impl/pb/ContainerStartDataPBImpl.java | 258 ---
.../timeline/GenericObjectMapper.java | 135 --
.../timeline/NameValuePair.java | 59 -
.../timeline/TimelineReader.java | 155 --
.../timeline/TimelineStore.java | 29 -
.../timeline/TimelineWriter.java | 46 -
.../webapp/AMSController.java | 37 -
.../webapp/AMSWebApp.java | 42 -
.../webapp/JAXBContextResolver.java | 64 -
.../webapp/TimelineWebServices.java | 513 -----
.../src/main/python/main.py | 4 +-
.../core/loadsimulator/data/TestAppMetrics.java | 134 ++
.../core/loadsimulator/data/TestMetric.java | 80 +
.../jmetertest/AMSJMeterLoadTest.java | 198 ++
.../jmetertest/jmetertest/AppGetMetric.java | 57 +
.../jmetertest/GetMetricRequestInfo.java | 61 +
.../jmetertest/JmeterTestPlanTask.java | 276 +++
.../net/TestRestMetricsSender.java | 75 +
.../net/TestStdOutMetricsSender.java | 37 +
.../util/TestRandomMetricsProvider.java | 36 +
.../util/TestTimeStampProvider.java | 51 +
.../timeline/AbstractMiniHBaseClusterTest.java | 293 +++
.../AbstractPhoenixConnectionlessTest.java | 111 +
.../metrics/core/timeline/FunctionTest.java | 62 +
.../HBaseTimelineMetricsServiceTest.java | 136 ++
.../core/timeline/ITPhoenixHBaseAccessor.java | 472 +++++
.../metrics/core/timeline/MetricTestHelper.java | 119 ++
.../core/timeline/PhoenixHBaseAccessorTest.java | 291 +++
.../metrics/core/timeline/TestClusterSuite.java | 34 +
.../core/timeline/TestMetricHostAggregate.java | 65 +
.../core/timeline/TestPhoenixTransactSQL.java | 608 ++++++
.../core/timeline/TestTimelineMetricStore.java | 123 ++
.../TimelineMetricStoreWatcherTest.java | 108 +
.../TimelineMetricsAggregatorMemorySink.java | 141 ++
.../timeline/TimelineMetricsFilterTest.java | 223 ++
.../TimelineMetricsIgniteCacheTest.java | 240 +++
.../core/timeline/TopNConditionTest.java | 105 +
.../AbstractTimelineAggregatorTest.java | 181 ++
.../timeline/aggregators/DownSamplerTest.java | 97 +
.../aggregators/ITClusterAggregator.java | 711 +++++++
.../aggregators/ITMetricAggregator.java | 355 ++++
...melineMetricClusterAggregatorSecondTest.java | 404 ++++
...sterAggregatorSecondWithCacheSourceTest.java | 114 ++
.../MetricCollectorHAControllerTest.java | 106 +
.../timeline/discovery/TestMetadataManager.java | 234 +++
.../timeline/discovery/TestMetadataSync.java | 126 ++
...elineMetricsSeriesAggregateFunctionTest.java | 188 ++
.../timeline/query/DefaultConditionTest.java | 170 ++
.../timeline/source/RawMetricsSourceTest.java | 141 ++
.../uuid/TimelineMetricUuidManagerTest.java | 175 ++
.../timeline/TestGenericObjectMapper.java | 102 +
.../metrics/webapp/TestTimelineWebServices.java | 118 ++
.../loadsimulator/data/TestAppMetrics.java | 134 --
.../metrics/loadsimulator/data/TestMetric.java | 80 -
.../jmetertest/AMSJMeterLoadTest.java | 198 --
.../jmetertest/jmetertest/AppGetMetric.java | 57 -
.../jmetertest/GetMetricRequestInfo.java | 61 -
.../jmetertest/JmeterTestPlanTask.java | 276 ---
.../net/TestRestMetricsSender.java | 76 -
.../net/TestStdOutMetricsSender.java | 37 -
.../util/TestRandomMetricsProvider.java | 36 -
.../util/TestTimeStampProvider.java | 51 -
.../timeline/AbstractMiniHBaseClusterTest.java | 295 ---
.../AbstractPhoenixConnectionlessTest.java | 111 -
.../metrics/timeline/FunctionTest.java | 62 -
.../HBaseTimelineMetricsServiceTest.java | 136 --
.../timeline/ITPhoenixHBaseAccessor.java | 439 ----
.../metrics/timeline/MetricTestHelper.java | 121 --
.../timeline/PhoenixHBaseAccessorTest.java | 292 ---
.../metrics/timeline/TestClusterSuite.java | 34 -
.../timeline/TestMetricHostAggregate.java | 66 -
.../timeline/TestPhoenixTransactSQL.java | 608 ------
.../timeline/TestTimelineMetricStore.java | 124 --
.../TimelineMetricStoreWatcherTest.java | 109 -
.../TimelineMetricsAggregatorMemorySink.java | 141 --
.../timeline/TimelineMetricsFilterTest.java | 223 --
.../TimelineMetricsIgniteCacheTest.java | 240 ---
.../metrics/timeline/TopNConditionTest.java | 105 -
.../AbstractTimelineAggregatorTest.java | 181 --
.../timeline/aggregators/DownSamplerTest.java | 97 -
.../aggregators/ITClusterAggregator.java | 669 ------
.../aggregators/ITMetricAggregator.java | 355 ----
...melineMetricClusterAggregatorSecondTest.java | 404 ----
...sterAggregatorSecondWithCacheSourceTest.java | 115 --
.../MetricCollectorHAControllerTest.java | 106 -
.../timeline/discovery/TestMetadataManager.java | 234 ---
.../timeline/discovery/TestMetadataSync.java | 126 --
...elineMetricsSeriesAggregateFunctionTest.java | 188 --
.../timeline/query/DefaultConditionTest.java | 170 --
.../timeline/source/RawMetricsSourceTest.java | 141 --
.../uuid/TimelineMetricUuidManagerTest.java | 184 --
.../timeline/TestGenericObjectMapper.java | 102 -
.../webapp/TestTimelineWebServices.java | 124 --
.../src/test/resources/loadsimulator/README | 2 +-
315 files changed, 25126 insertions(+), 25184 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/Precision.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/Precision.java b/ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/Precision.java
index e87f06e..39bcce0 100644
--- a/ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/Precision.java
+++ b/ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/Precision.java
@@ -20,7 +20,6 @@ package org.apache.hadoop.metrics2.sink.timeline;
/**
* Is used to determine metrics aggregate table.
*
- * @see org.apache.hadoop.yarn.server.applicationhistoryservice.webapp.TimelineWebServices#getTimelineMetric
*/
public enum Precision {
SECONDS,
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/conf/unix/ambari-metrics-collector
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/conf/unix/ambari-metrics-collector b/ambari-metrics/ambari-metrics-timelineservice/conf/unix/ambari-metrics-collector
index de764ec..58984a1 100644
--- a/ambari-metrics/ambari-metrics-timelineservice/conf/unix/ambari-metrics-collector
+++ b/ambari-metrics/ambari-metrics-timelineservice/conf/unix/ambari-metrics-collector
@@ -238,7 +238,7 @@ function start()
echo "$(date) Launching in distributed mode. Assuming Hbase daemons up and running." | tee -a $STARTUPFILE
fi
- CLASS='org.apache.hadoop.yarn.server.applicationhistoryservice.AMSApplicationServer'
+ CLASS='org.apache.ambari.metrics.AMSApplicationServer'
# YARN_OPTS="${YARN_OPTS} ${YARN_TIMELINESERVER_OPTS}"
# if [[ -n "${YARN_TIMELINESERVER_HEAPSIZE}" ]]; then
# JAVA_HEAP_MAX="-Xmx${YARN_TIMELINESERVER_HEAPSIZE}m"
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/pom.xml
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/pom.xml b/ambari-metrics/ambari-metrics-timelineservice/pom.xml
index d3a976b..35aa79b 100644
--- a/ambari-metrics/ambari-metrics-timelineservice/pom.xml
+++ b/ambari-metrics/ambari-metrics-timelineservice/pom.xml
@@ -857,7 +857,7 @@
<classpathPrefix></classpathPrefix>
<!-- Specifies the main class of the application -->
<mainClass>
- org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.loadsimulator.MetricsLoadSimulator
+ org.apache.ambari.metrics.core.loadsimulator.MetricsLoadSimulator
</mainClass>
</manifest>
</archive>
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/AMSApplicationServer.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/AMSApplicationServer.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/AMSApplicationServer.java
new file mode 100644
index 0000000..b39135e
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/AMSApplicationServer.java
@@ -0,0 +1,141 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.metrics;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.http.HttpConfig;
+import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
+import org.apache.hadoop.metrics2.source.JvmMetrics;
+import org.apache.hadoop.service.CompositeService;
+import org.apache.hadoop.util.ExitUtil;
+import org.apache.hadoop.util.ShutdownHookManager;
+import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.yarn.YarnUncaughtExceptionHandler;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
+import org.apache.ambari.metrics.core.timeline.HBaseTimelineMetricsService;
+import org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguration;
+import org.apache.ambari.metrics.core.timeline.TimelineMetricStore;
+import org.apache.ambari.metrics.webapp.AMSWebApp;
+import org.apache.hadoop.yarn.webapp.WebApp;
+import org.apache.hadoop.yarn.webapp.WebApps;
+
+/**
+ * Metrics collector web server
+ */
+public class AMSApplicationServer extends CompositeService {
+
+ public static final int SHUTDOWN_HOOK_PRIORITY = 30;
+ private static final Log LOG = LogFactory.getLog(AMSApplicationServer.class);
+
+ TimelineMetricStore timelineMetricStore;
+ private WebApp webApp;
+ private TimelineMetricConfiguration metricConfiguration;
+
+ public AMSApplicationServer() {
+ super(AMSApplicationServer.class.getName());
+ }
+
+ @Override
+ protected void serviceInit(Configuration conf) throws Exception {
+ metricConfiguration = TimelineMetricConfiguration.getInstance();
+ metricConfiguration.initialize();
+ timelineMetricStore = createTimelineMetricStore(conf);
+ addIfService(timelineMetricStore);
+ super.serviceInit(conf);
+ }
+
+ @Override
+ protected void serviceStart() throws Exception {
+ DefaultMetricsSystem.initialize("AmbariMetricsSystem");
+ JvmMetrics.initSingleton("AmbariMetricsSystem", null);
+
+ startWebApp();
+ super.serviceStart();
+ }
+
+ @Override
+ protected void serviceStop() throws Exception {
+ if (webApp != null) {
+ webApp.stop();
+ }
+
+ DefaultMetricsSystem.shutdown();
+ super.serviceStop();
+ }
+
+ static AMSApplicationServer launchAMSApplicationServer(String[] args) {
+ Thread.setDefaultUncaughtExceptionHandler(new YarnUncaughtExceptionHandler());
+ StringUtils.startupShutdownMessage(AMSApplicationServer.class, args, LOG);
+ AMSApplicationServer amsApplicationServer = null;
+ try {
+ amsApplicationServer = new AMSApplicationServer();
+ ShutdownHookManager.get().addShutdownHook(
+ new CompositeServiceShutdownHook(amsApplicationServer),
+ SHUTDOWN_HOOK_PRIORITY);
+ YarnConfiguration conf = new YarnConfiguration();
+ amsApplicationServer.init(conf);
+ amsApplicationServer.start();
+ } catch (Throwable t) {
+ LOG.fatal("Error starting AMSApplicationServer", t);
+ ExitUtil.terminate(-1, "Error starting AMSApplicationServer");
+ }
+ return amsApplicationServer;
+ }
+
+ public static void main(String[] args) {
+ launchAMSApplicationServer(args);
+ }
+
+ protected TimelineMetricStore createTimelineMetricStore(Configuration conf) {
+ LOG.info("Creating metrics store.");
+ return new HBaseTimelineMetricsService(metricConfiguration);
+ }
+
+ protected void startWebApp() {
+ String bindAddress = null;
+ try {
+ bindAddress = metricConfiguration.getWebappAddress();
+ } catch (Exception e) {
+ throw new ExceptionInInitializerError("Cannot find bind address");
+ }
+ LOG.info("Instantiating metrics collector at " + bindAddress);
+ try {
+ Configuration conf = metricConfiguration.getMetricsConf();
+ conf.set("hadoop.http.max.threads", String.valueOf(metricConfiguration
+ .getTimelineMetricsServiceHandlerThreadCount()));
+ HttpConfig.Policy policy = HttpConfig.Policy.valueOf(
+ conf.get(TimelineMetricConfiguration.TIMELINE_SERVICE_HTTP_POLICY,
+ HttpConfig.Policy.HTTP_ONLY.name()));
+ webApp =
+ WebApps
+ .$for("timeline", null, null, "ws")
+ .withHttpPolicy(conf, policy)
+ .at(bindAddress)
+ .start(new AMSWebApp(timelineMetricStore));
+ } catch (Exception e) {
+ String msg = "AHSWebApp failed to start.";
+ LOG.error(msg, e);
+ throw new YarnRuntimeException(msg, e);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/loadsimulator/LoadRunner.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/loadsimulator/LoadRunner.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/loadsimulator/LoadRunner.java
new file mode 100644
index 0000000..679339b
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/loadsimulator/LoadRunner.java
@@ -0,0 +1,154 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.metrics.core.loadsimulator;
+
+import static org.apache.ambari.metrics.core.loadsimulator.data.AppID.MASTER_APPS;
+import static org.apache.ambari.metrics.core.loadsimulator.data.AppID.SLAVE_APPS;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Date;
+import java.util.List;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.ambari.metrics.core.loadsimulator.data.ApplicationInstance;
+import org.apache.ambari.metrics.core.loadsimulator.data.HostMetricsGenerator;
+import org.apache.ambari.metrics.core.loadsimulator.data.MetricsGeneratorConfigurer;
+import org.apache.ambari.metrics.core.loadsimulator.net.MetricsSender;
+import org.apache.ambari.metrics.core.loadsimulator.net.RestMetricsSender;
+import org.apache.ambari.metrics.core.loadsimulator.util.TimeStampProvider;
+import org.apache.ambari.metrics.core.loadsimulator.data.AppID;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ *
+ */
+public class LoadRunner {
+ private final static Logger LOG = LoggerFactory.getLogger(LoadRunner.class);
+
+ private final ScheduledExecutorService timer;
+ private final ExecutorService workersPool;
+ private final Collection<Callable<String>> workers;
+ private final long startTime = new Date().getTime();
+ private final int collectIntervalMillis;
+ private final int sendIntervalMillis;
+
+ public LoadRunner(String hostName,
+ int threadCount,
+ String metricsHostName,
+ int minHostIndex,
+ int collectIntervalMillis,
+ int sendIntervalMillis,
+ boolean createMaster) {
+ this.collectIntervalMillis = collectIntervalMillis;
+ this.workersPool = Executors.newFixedThreadPool(threadCount);
+ this.timer = Executors.newScheduledThreadPool(1);
+ this.sendIntervalMillis = sendIntervalMillis;
+
+ workers = prepareWorkers(hostName, threadCount, metricsHostName, createMaster, minHostIndex);
+ }
+
+ private Collection<Callable<String>> prepareWorkers(String hostName,
+ int threadCount,
+ String metricsHost,
+ Boolean createMaster, int minHostIndex) {
+ Collection<Callable<String>> senderWorkers =
+ new ArrayList<Callable<String>>(threadCount);
+
+ int startIndex = minHostIndex;
+ if (createMaster) {
+ String simHost = hostName + startIndex;
+ addMetricsWorkers(senderWorkers, simHost, metricsHost, MASTER_APPS);
+ startIndex++;
+ }
+
+ for (int i = startIndex; i < threadCount + minHostIndex; i++) {
+ String simHost = hostName + i;
+ addMetricsWorkers(senderWorkers, simHost, metricsHost, SLAVE_APPS);
+ }
+
+ return senderWorkers;
+ }
+
+ private void addMetricsWorkers(Collection<Callable<String>> senderWorkers,
+ String specificHostName,
+ String metricsHostName,
+ AppID[] apps) {
+ for (AppID app : apps) {
+ HostMetricsGenerator metricsGenerator =
+ createApplicationMetrics(specificHostName, app);
+ MetricsSender sender = new RestMetricsSender(metricsHostName);
+ senderWorkers.add(new MetricsSenderWorker(sender, metricsGenerator));
+ }
+ }
+
+ private HostMetricsGenerator createApplicationMetrics(String simHost, AppID host) {
+ ApplicationInstance appInstance = new ApplicationInstance(simHost, host, "");
+ TimeStampProvider timeStampProvider = new TimeStampProvider(startTime,
+ collectIntervalMillis, sendIntervalMillis);
+
+ return MetricsGeneratorConfigurer
+ .createMetricsForHost(appInstance, timeStampProvider);
+ }
+
+ public void start() {
+ timer.scheduleAtFixedRate(new Runnable() {
+ @Override
+ public void run() {
+ try {
+ runOnce();
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ }
+ }
+ }, 0, sendIntervalMillis, TimeUnit.MILLISECONDS);
+ }
+
+ public void runOnce() throws InterruptedException {
+ List<Future<String>> futures = workersPool.invokeAll(workers,
+ sendIntervalMillis / 2,
+ TimeUnit.MILLISECONDS);
+ int done = 0;
+
+ // TODO: correctly count the failed tasks
+ for (Future<String> future : futures) {
+ done += future.isDone() ? 1 : 0;
+ }
+
+ LOG.info("Finished successfully " + done + " tasks ");
+ }
+
+ public void shutdown() {
+ timer.shutdownNow();
+ workersPool.shutdownNow();
+ }
+
+ public static void main(String[] args) {
+ LoadRunner runner =
+ new LoadRunner("local", 0, "metrics", 0, 10000, 20000, false);
+
+ runner.start();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/loadsimulator/MetricsLoadSimulator.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/loadsimulator/MetricsLoadSimulator.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/loadsimulator/MetricsLoadSimulator.java
new file mode 100644
index 0000000..93e03a5
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/loadsimulator/MetricsLoadSimulator.java
@@ -0,0 +1,138 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.metrics.core.loadsimulator;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Sample Usage:
+ * <pre>
+ * $ java -cp "dependency/*":LoadSimulator-1.0-SNAPSHOT.jar \
+ * org.apache.ambari.metrics.MetricsLoadSimulator \
+ * -h "bartosz.laptop" -n 2 -m "162.216.148.45" -c 10000 -s 30000</pre>
+ */
+public class MetricsLoadSimulator {
+ private final static Logger LOG = LoggerFactory.getLogger(MetricsLoadSimulator
+ .class);
+
+ public static void main(String[] args) throws IOException, InterruptedException {
+ Map<String, String> mapArgs = parseArgs(args);
+
+ LoadRunner loadRunner = new LoadRunner(
+ mapArgs.get("hostName"),
+ Integer.valueOf(mapArgs.get("numberOfHosts")),
+ mapArgs.get("metricsHostName"),
+ mapArgs.get("minHostIndex") == null ? 0 : Integer.valueOf(mapArgs.get("minHostIndex")),
+ Integer.valueOf(mapArgs.get("collectInterval")),
+ Integer.valueOf(mapArgs.get("sendInterval")),
+ Boolean.valueOf(mapArgs.get("master"))
+ );
+
+ loadRunner.start();
+ }
+
+ /*
+ Another entry point to the test to be called from ./jmetertest/AMSJMeterLoadTest.java
+ */
+ public static void startTest(Map<String,String> mapArgs) {
+
+ LoadRunner loadRunner = new LoadRunner(
+ mapArgs.get("hostName"),
+ Integer.valueOf(mapArgs.get("numberOfHosts")),
+ mapArgs.get("metricsHostName"),
+ mapArgs.get("minHostIndex") == null ? 0 : Integer.valueOf(mapArgs.get("minHostIndex")),
+ Integer.valueOf(mapArgs.get("collectInterval")),
+ Integer.valueOf(mapArgs.get("sendInterval")),
+ Boolean.valueOf(mapArgs.get("master"))
+ );
+
+ loadRunner.start();
+ }
+
+ private static Map<String, String> parseArgs(String[] args) {
+ Map<String, String> mapProps = new HashMap<String, String>();
+ mapProps.put("hostName", "host");
+ mapProps.put("numberOfHosts", "20");
+ mapProps.put("trafficType", "burst");
+ mapProps.put("metricsHostName", "localhost");
+ mapProps.put("collectInterval", "10000");
+ mapProps.put("sendInterval", "60000");
+
+ if (args.length == 0) {
+ printUsage();
+ throw new RuntimeException("Unexpected argument, See usage message.");
+ } else {
+ for (int i = 0; i < args.length; i += 2) {
+ String arg = args[i];
+ if (arg.equals("-h")) {
+ mapProps.put("hostName", args[i + 1]);
+ } else if (arg.equals("-n")) {
+ mapProps.put("numberOfHosts", args[i + 1]);
+ } else if (arg.equals("-t")) {
+ mapProps.put("trafficType", args[i + 1]);
+ } else if (arg.equals("-m")) {
+ mapProps.put("metricsHostName", args[i + 1]);
+ } else if (arg.equals("-c")) {
+ mapProps.put("collectInterval", args[i + 1]);
+ } else if (arg.equals("-s")) {
+ mapProps.put("sendInterval", args[i + 1]);
+ } else if (arg.equals("-M")) {
+ mapProps.put("master", args[i + 1]);
+ } else if (arg.equals("-d")) {
+ // a dummy switch - it says that we agree with defaults
+ } else {
+ printUsage();
+ throw new RuntimeException("Unexpected argument, See usage message.");
+ }
+ }
+ }
+
+ LOG.info("Recognized options: baseHostName={} hosts#={} trafficMode={} " +
+ "metricsHostName={} collectIntervalMillis={} sendIntervalMillis={} " +
+ "simulateMaster={}",
+ mapProps.get("hostName"),
+ Integer.valueOf(mapProps.get("numberOfHosts")),
+ mapProps.get("trafficType"),
+ mapProps.get("metricsHostName"),
+ Integer.valueOf(mapProps.get("collectInterval")),
+ Integer.valueOf(mapProps.get("sendInterval")),
+ Boolean.valueOf(mapProps.get("master"))
+ );
+
+ return mapProps;
+ }
+
+ public static void printUsage() {
+ System.err.println("Usage: java MetricsLoadSimulator [OPTIONS]");
+ System.err.println("Options: ");
+ System.err.println("[-h hostName] [-n numberOfHosts] "
+ + "[-t trafficMode {burst, staggered}] [-m metricsHostName] "
+ + "[-c collectIntervalMillis {10 sec}] [-s sendIntervalMillis {60 sec}]"
+ + "[-M simulateMaster {true, false}] ");
+ System.err.println();
+ System.err.println("When you select a master, then one simulated host will play");
+ System.err.println("a role of a master, and the rest will be slaves. Otherwise");
+ System.err.println("all simulation threads (single thread is for single host)");
+ System.err.println("will be slave hosts");
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/loadsimulator/MetricsSenderWorker.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/loadsimulator/MetricsSenderWorker.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/loadsimulator/MetricsSenderWorker.java
new file mode 100644
index 0000000..6e5e6b7
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/loadsimulator/MetricsSenderWorker.java
@@ -0,0 +1,60 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.metrics.core.loadsimulator;
+
+
+import java.io.IOException;
+import java.util.concurrent.Callable;
+
+import org.apache.ambari.metrics.core.loadsimulator.util.Json;
+import org.apache.ambari.metrics.core.loadsimulator.data.AppMetrics;
+import org.apache.ambari.metrics.core.loadsimulator.data.HostMetricsGenerator;
+import org.apache.ambari.metrics.core.loadsimulator.net.MetricsSender;
+import org.apache.ambari.metrics.core.loadsimulator.net.RestMetricsSender;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ */
+public class MetricsSenderWorker implements Callable<String> {
+ private final static Logger LOG = LoggerFactory.getLogger(RestMetricsSender.class);
+
+ MetricsSender sender;
+ HostMetricsGenerator hmg;
+
+ public MetricsSenderWorker(MetricsSender sender, HostMetricsGenerator metricsGenerator) {
+ this.sender = sender;
+ hmg = metricsGenerator;
+ }
+
+ @Override
+ public String call() throws Exception {
+ AppMetrics hostMetrics = hmg.createMetrics();
+
+ try {
+ String request = new Json().serialize(hostMetrics); //inject?
+ String response = sender.pushMetrics(request);
+
+ return response;
+ } catch (IOException e) {
+ LOG.error("Error while pushing metrics: ", e);
+ throw e;
+ }
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/loadsimulator/data/AppID.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/loadsimulator/data/AppID.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/loadsimulator/data/AppID.java
new file mode 100644
index 0000000..7290516
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/loadsimulator/data/AppID.java
@@ -0,0 +1,45 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.metrics.core.loadsimulator.data;
+
+public enum AppID {
+ HOST("HOST"),
+ NAMENODE("namenode"),
+ RESOURCEMANAGER("resourcemanager"),
+ DATANODE("datanode"),
+ NODEMANAGER("nodemanager"),
+ MASTER_HBASE("hbase"),
+ SLAVE_HBASE("hbase"),
+ NIMBUS("nimbus"),
+ HIVEMETASTORE("hivemetastore"),
+ HIVESERVER2("hiveserver2"),
+ KAFKA_BROKER("kafka_broker");
+
+ public static final AppID[] MASTER_APPS = {HOST, NAMENODE, RESOURCEMANAGER, MASTER_HBASE, KAFKA_BROKER, NIMBUS, HIVEMETASTORE, HIVESERVER2};
+ public static final AppID[] SLAVE_APPS = {HOST, DATANODE, NODEMANAGER, SLAVE_HBASE};
+
+ private String id;
+
+ private AppID(String id) {
+ this.id = id;
+ }
+
+ public String getId() {
+ return id;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/loadsimulator/data/AppMetrics.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/loadsimulator/data/AppMetrics.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/loadsimulator/data/AppMetrics.java
new file mode 100644
index 0000000..d49c81c
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/loadsimulator/data/AppMetrics.java
@@ -0,0 +1,47 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.metrics.core.loadsimulator.data;
+
+import java.util.ArrayList;
+import java.util.Collection;
+
+/**
+ * AppMetrics is a class that helps to create properly initialized metrics for
+ * current app. It also holds the
+ * metrics and can be serialized to json.
+ */
+public class AppMetrics {
+
+ private final Collection<Metric> metrics = new ArrayList<Metric>();
+ private final transient ApplicationInstance applicationId;
+ private final transient long startTime;
+
+ public AppMetrics(ApplicationInstance applicationId, long startTime) {
+ this.applicationId = applicationId;
+ this.startTime = startTime;
+ }
+
+ public Metric createMetric(String metricName) {
+ return new Metric(applicationId, metricName, startTime);
+ }
+
+ public void addMetric(Metric metric) {
+ metrics.add(metric);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/loadsimulator/data/ApplicationInstance.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/loadsimulator/data/ApplicationInstance.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/loadsimulator/data/ApplicationInstance.java
new file mode 100644
index 0000000..ffe942b
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/loadsimulator/data/ApplicationInstance.java
@@ -0,0 +1,58 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.metrics.core.loadsimulator.data;
+
+/**
+ * AppId is a helper class that encapsulates the common part of metrics message.
+ * It contains hostName, appId and instanceId. It is immutable,
+ * and it can not hold null values.
+ */
+public final class ApplicationInstance {
+
+ private final transient String hostName;
+ private final transient AppID appId;
+ private final transient String instanceId;
+
+ /**
+ * @param hostname
+ * @param appId
+ * @param instanceId
+ */
+ public ApplicationInstance(String hostname, AppID appId, String instanceId) {
+ if (hostname == null || appId == null || instanceId == null)
+ throw new IllegalArgumentException("ApplicationInstance can not be " +
+ "instantiated with null values");
+
+ this.hostName = hostname;
+ this.appId = appId;
+ this.instanceId = instanceId;
+ }
+
+ public String getInstanceId() {
+ return instanceId;
+ }
+
+ public AppID getAppId() {
+ return appId;
+ }
+
+ public String getHostName() {
+ return hostName;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/loadsimulator/data/HostMetricsGenerator.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/loadsimulator/data/HostMetricsGenerator.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/loadsimulator/data/HostMetricsGenerator.java
new file mode 100644
index 0000000..7e2fd1a
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/loadsimulator/data/HostMetricsGenerator.java
@@ -0,0 +1,61 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.metrics.core.loadsimulator.data;
+
+
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.ambari.metrics.core.loadsimulator.util.RandomMetricsProvider;
+import org.apache.ambari.metrics.core.loadsimulator.util.TimeStampProvider;
+
+/**
+ */
+public class HostMetricsGenerator {
+
+ private Map<String, RandomMetricsProvider> metricDataProviders = new HashMap<String, RandomMetricsProvider>();
+ private final TimeStampProvider tsp;
+ private final ApplicationInstance id;
+
+ public HostMetricsGenerator(ApplicationInstance id,
+ TimeStampProvider timeStamps,
+ Map<String, RandomMetricsProvider> metricDataProviders) {
+ this.id = id;
+ this.tsp = timeStamps;
+ this.metricDataProviders = metricDataProviders;
+ }
+
+ public AppMetrics createMetrics() {
+ long[] timestamps = tsp.timestampsForNextInterval();
+ AppMetrics appMetrics = new AppMetrics(id, timestamps[0]);
+
+ for (Map.Entry<String, RandomMetricsProvider> entry : metricDataProviders.entrySet()) {
+ String metricName = entry.getKey();
+ RandomMetricsProvider metricData = entry.getValue();
+
+ Metric metric = appMetrics.createMetric(metricName);
+ for (long timestamp : timestamps) {
+ metric.putMetric(timestamp, String.valueOf(metricData.next()));
+ }
+ appMetrics.addMetric(metric);
+ }
+
+ return appMetrics;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/loadsimulator/data/Metric.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/loadsimulator/data/Metric.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/loadsimulator/data/Metric.java
new file mode 100644
index 0000000..7bcc852
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/loadsimulator/data/Metric.java
@@ -0,0 +1,71 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.metrics.core.loadsimulator.data;
+
+import java.util.LinkedHashMap;
+import java.util.Map;
+
+public class Metric {
+
+ private String instanceid;
+ private String hostname;
+ private Map<String, String> metrics = new LinkedHashMap<String, String>();
+ private String starttime;
+ private String appid;
+ private String metricname;
+
+ // i don't like this ctor, but it has to be public for json deserialization
+ public Metric() {
+ }
+
+ public Metric(ApplicationInstance app, String metricName, long startTime) {
+ this.hostname = app.getHostName();
+ this.appid = app.getAppId().getId();
+ this.instanceid = app.getInstanceId();
+ this.metricname = metricName;
+ this.starttime = Long.toString(startTime);
+ }
+
+ public void putMetric(long timestamp, String value) {
+ metrics.put(Long.toString(timestamp), value);
+ }
+
+ public String getInstanceid() {
+ return instanceid;
+ }
+
+ public String getHostname() {
+ return hostname;
+ }
+
+ public Map<String, String> getMetrics() {
+ return metrics;
+ }
+
+ public String getStarttime() {
+ return starttime;
+ }
+
+ public String getAppid() {
+ return appid;
+ }
+
+ public String getMetricname() {
+ return metricname;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/loadsimulator/data/MetricsGeneratorConfigurer.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/loadsimulator/data/MetricsGeneratorConfigurer.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/loadsimulator/data/MetricsGeneratorConfigurer.java
new file mode 100644
index 0000000..4f848a4
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/loadsimulator/data/MetricsGeneratorConfigurer.java
@@ -0,0 +1,93 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.metrics.core.loadsimulator.data;
+
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.ambari.metrics.core.loadsimulator.util.TimeStampProvider;
+import org.apache.ambari.metrics.core.loadsimulator.util.RandomMetricsProvider;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * MetricsGeneratorConfigurer is a factory that reads metrics definition from a file,
+ * and returns an Single HostMetricsGenerator. Check createMetricsForHost method
+ * for details.
+ */
+public class MetricsGeneratorConfigurer {
+
+ private final static Logger LOG = LoggerFactory.getLogger
+ (MetricsGeneratorConfigurer.class);
+
+ /**
+ * Creates HostMetricsGenerator configured with metric names loaded from file.
+ *
+ * @param id ApplicationInstance descriptor, will be used to create
+ * HostMetricsGenerator, cannot be null
+ * @param timeStamps configured TimeStampProvider that can provide next
+ * timestamp, cannot be null
+ * @return HostMetricsGenerator with given ApplicationInstance id and configured
+ * mapping of
+ * metric names to data providers
+ */
+ public static HostMetricsGenerator createMetricsForHost(
+ ApplicationInstance id,
+ TimeStampProvider timeStamps) {
+ return new HostMetricsGenerator(id, timeStamps, readMetrics(id.getAppId()));
+ }
+
+ private static Map<String, RandomMetricsProvider> readMetrics(AppID type) {
+ InputStream input = null;
+ Map<String, RandomMetricsProvider> metrics =
+ new HashMap<String, RandomMetricsProvider>();
+ String fileName = "metrics_def/" + type.toString() + ".dat";
+
+ try {
+ LOG.info("Loading " + fileName);
+
+ input = MetricsGeneratorConfigurer.class.getClassLoader()
+ .getResourceAsStream(fileName);
+
+ BufferedReader reader = new BufferedReader(new InputStreamReader(input));
+
+ String line;
+ while ((line = reader.readLine()) != null) {
+ metrics.put(line.trim(), new RandomMetricsProvider(100, 200));
+ }
+
+ } catch (IOException e) {
+ LOG.error("Cannot read file " + fileName + " for appID " + type.toString(), e);
+ } finally {
+ if (input != null) {
+ try {
+ input.close();
+ } catch (IOException ex) {
+ // intentionally left blank, here we cannot do anything
+ }
+ }
+ }
+
+ return metrics;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/loadsimulator/net/MetricsSender.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/loadsimulator/net/MetricsSender.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/loadsimulator/net/MetricsSender.java
new file mode 100644
index 0000000..e3876c7
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/loadsimulator/net/MetricsSender.java
@@ -0,0 +1,31 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.metrics.core.loadsimulator.net;
+
+/**
+ * MetricSender should provides a simple way of pushing metrics to some service.
+ */
+public interface MetricsSender {
+ /**
+ * Push metrics to the metric service (e.g. a metrics storage system).
+ *
+ * @param payload the payload to be sent to metrics service
+ * @return response message either acknowledgement or error
+ */
+ String pushMetrics(String payload);
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/loadsimulator/net/RestMetricsSender.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/loadsimulator/net/RestMetricsSender.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/loadsimulator/net/RestMetricsSender.java
new file mode 100644
index 0000000..455b381
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/loadsimulator/net/RestMetricsSender.java
@@ -0,0 +1,92 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.metrics.core.loadsimulator.net;
+
+import java.io.IOException;
+import java.util.concurrent.TimeUnit;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.base.Stopwatch;
+
+/**
+ * Implements MetricsSender and provides a way of pushing metrics to application metrics history service using REST
+ * endpoint.
+ */
+public class RestMetricsSender implements MetricsSender {
+ private final static Logger LOG = LoggerFactory.getLogger(RestMetricsSender.class);
+
+ private final static String COLLECTOR_URL = "http://%s/ws/v1/timeline/metrics";
+ private final String collectorServiceAddress;
+
+ /**
+ * Creates unconnected RestMetricsSender with endpoint configured as
+ * http://${metricsHost}:6188/ws/v1/timeline/metrics,
+ * where ${metricsHost} is specified by metricHost param.
+ *
+ * @param metricsHost the hostname that will be used to access application metrics history service.
+ */
+ public RestMetricsSender(String metricsHost) {
+ collectorServiceAddress = String.format(COLLECTOR_URL, metricsHost);
+ }
+
+ /**
+ * Push metrics to the REST endpoint. Connection is always open and closed on every call.
+ *
+ * @param payload the payload with metrics to be sent to metrics service
+ * @return response message either acknowledgement or error, empty on exception
+ */
+ @Override
+ public String pushMetrics(String payload) {
+ String responseString = "";
+ UrlService svc = null;
+ Stopwatch timer = new Stopwatch().start();
+
+ try {
+ LOG.info("server: {}", collectorServiceAddress);
+
+ svc = getConnectedUrlService();
+ responseString = svc.send(payload);
+
+ timer.stop();
+ LOG.info("http response time: " + timer.elapsed(TimeUnit.MILLISECONDS)
+ + " ms");
+
+ if (responseString.length() > 0) {
+ LOG.debug("POST response from server: " + responseString);
+ }
+ } catch (IOException e) {
+ LOG.error("", e);
+ } finally {
+ if (svc != null) {
+ svc.disconnect();
+ }
+ }
+
+ return responseString;
+ }
+
+ /**
+ * Relaxed to protected for testing.
+ */
+ protected UrlService getConnectedUrlService() throws IOException {
+ return UrlService.newConnection(collectorServiceAddress);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/loadsimulator/net/StdOutMetricsSender.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/loadsimulator/net/StdOutMetricsSender.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/loadsimulator/net/StdOutMetricsSender.java
new file mode 100644
index 0000000..335b2c3
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/loadsimulator/net/StdOutMetricsSender.java
@@ -0,0 +1,56 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.metrics.core.loadsimulator.net;
+
+import java.io.PrintStream;
+
+/**
+ * StdOutMetricsSender dumps metrics to defined PrintStream out. It is useful for testing.
+ */
+public class StdOutMetricsSender implements MetricsSender {
+ public final PrintStream out;
+ private String metricsHostName;
+
+ /**
+ * Creates new StdOutMetricsSender with specified hostname (only used in messages) and sends output to System.out
+ *
+ * @param metricsHostName a name used in printed messages
+ */
+ public StdOutMetricsSender(String metricsHostName) {
+ this(metricsHostName, System.out);
+ }
+
+ /**
+ * Creates new StdOutMetricsSender with specified hostname (only used in messages) and PrintStream which is used as
+ * an output.
+ *
+ * @param metricsHostName a name used in printed messages
+ * @param out PrintStream that the Sender will write to, can be System.out
+ */
+ public StdOutMetricsSender(String metricsHostName, PrintStream out) {
+ this.metricsHostName = metricsHostName;
+ this.out = out;
+ }
+
+ @Override
+ public String pushMetrics(String payload) {
+ out.println("Sending to " + metricsHostName + ": " + payload);
+
+ return "OK";
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/loadsimulator/net/UrlService.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/loadsimulator/net/UrlService.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/loadsimulator/net/UrlService.java
new file mode 100644
index 0000000..b30f854
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/loadsimulator/net/UrlService.java
@@ -0,0 +1,100 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.metrics.core.loadsimulator.net;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.OutputStreamWriter;
+import java.net.HttpURLConnection;
+import java.net.URL;
+
+public class UrlService {
+
+ public static final int CONNECT_TIMEOUT = 20000;
+ public static final int READ_TIMEOUT = 20000;
+ private final String address;
+ private HttpURLConnection conn;
+
+ private UrlService(String address) {
+ this.address = address;
+ }
+
+ /**
+ * Returns a new UrlService connected to specified address.
+ *
+ * @param address
+ * @return
+ * @throws IOException
+ */
+ public static UrlService newConnection(String address) throws IOException {
+ UrlService svc = new UrlService(address);
+ svc.connect();
+
+ return svc;
+ }
+
+ public HttpURLConnection connect() throws IOException {
+ URL url = new URL(address);
+ conn = (HttpURLConnection) url.openConnection();
+
+ //TODO: make timeouts configurable
+ conn.setConnectTimeout(CONNECT_TIMEOUT);
+ conn.setReadTimeout(READ_TIMEOUT);
+ conn.setDoInput(true);
+ conn.setDoOutput(true);
+ conn.setRequestMethod("POST");
+ conn.setRequestProperty("Content-Type", "application/json");
+ conn.setRequestProperty("Accept", "*/*");
+
+ return conn;
+ }
+
+ public String send(String payload) throws IOException {
+ if (conn == null)
+ throw new IllegalStateException("Cannot use unconnected UrlService");
+ write(payload);
+
+ return read();
+ }
+
+ private String read() throws IOException {
+ StringBuilder response = new StringBuilder();
+
+ BufferedReader br = new BufferedReader(new InputStreamReader(
+ conn.getInputStream()));
+ String line = null;
+ while ((line = br.readLine()) != null) {
+ response.append(line);
+ }
+ br.close();
+
+ return response.toString();
+ }
+
+ private void write(String payload) throws IOException {
+ OutputStreamWriter writer = new OutputStreamWriter(conn.getOutputStream(),
+ "UTF-8");
+ writer.write(payload);
+ writer.close();
+ }
+
+ public void disconnect() {
+ conn.disconnect();
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/loadsimulator/util/Json.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/loadsimulator/util/Json.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/loadsimulator/util/Json.java
new file mode 100644
index 0000000..3e186a1
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/loadsimulator/util/Json.java
@@ -0,0 +1,62 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.metrics.core.loadsimulator.util;
+
+import java.io.IOException;
+
+import org.codehaus.jackson.annotate.JsonAutoDetect;
+import org.codehaus.jackson.annotate.JsonMethod;
+import org.codehaus.jackson.map.ObjectMapper;
+import org.codehaus.jackson.map.SerializationConfig;
+
+/**
+ * Small wrapper that configures the ObjectMapper with some defaults.
+ */
+public class Json {
+ private ObjectMapper myObjectMapper;
+
+ /**
+ * Creates default Json ObjectMapper that maps fields.
+ */
+ public Json() {
+ this(false);
+ }
+
+ /**
+ * Creates a Json ObjectMapper that maps fields and optionally pretty prints the
+ * serialized objects.
+ *
+ * @param pretty a flag - if true the output will be pretty printed.
+ */
+ public Json(boolean pretty) {
+ myObjectMapper = new ObjectMapper();
+ myObjectMapper.setVisibility(JsonMethod.FIELD, JsonAutoDetect.Visibility.ANY);
+ if (pretty) {
+ myObjectMapper.configure(SerializationConfig.Feature.INDENT_OUTPUT, true);
+ }
+ }
+
+ public String serialize(Object o) throws IOException {
+ return myObjectMapper.writeValueAsString(o);
+ }
+
+ public <T> T deserialize(String content, Class<T> paramClass) throws IOException {
+ return myObjectMapper.readValue(content, paramClass);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/loadsimulator/util/RandomMetricsProvider.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/loadsimulator/util/RandomMetricsProvider.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/loadsimulator/util/RandomMetricsProvider.java
new file mode 100644
index 0000000..a0cf9f6
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/loadsimulator/util/RandomMetricsProvider.java
@@ -0,0 +1,39 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.metrics.core.loadsimulator.util;
+
+import java.util.Random;
+
+/**
+ */
+public class RandomMetricsProvider {
+
+ private double min;
+ private double max;
+ private Random rnd;
+
+ public RandomMetricsProvider(double min, double max) {
+ this.min = min;
+ this.max = max;
+ this.rnd = new Random();
+ }
+
+ public double next() {
+ return rnd.nextDouble() * (max - min) + min;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/loadsimulator/util/TimeStampProvider.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/loadsimulator/util/TimeStampProvider.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/loadsimulator/util/TimeStampProvider.java
new file mode 100644
index 0000000..015a297
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/loadsimulator/util/TimeStampProvider.java
@@ -0,0 +1,51 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.metrics.core.loadsimulator.util;
+
+/**
+ */
+public class TimeStampProvider {
+ private int timeStep;
+ private long currentTime;
+ private int sendInterval;
+
+ public TimeStampProvider(long startTime, int timeStep, int sendInterval) {
+ this.timeStep = timeStep;
+ this.currentTime = startTime - timeStep;
+ this.sendInterval = sendInterval;
+ }
+
+ public long next() {
+ return currentTime += timeStep;
+ }
+
+ public long[] timestampsForNextInterval() {
+ return timestampsForInterval(sendInterval);
+ }
+
+ private long[] timestampsForInterval(int sendInterval) {
+ int steps = sendInterval / timeStep;
+ long[] timestamps = new long[steps];
+
+ for (int i = 0; i < timestamps.length; i++) {
+ timestamps[i] = next();
+ }
+
+ return timestamps;
+ }
+}
[10/28] ambari git commit: AMBARI-22740 : Fix integration test for
HBase in branch-3.0-ams due to UUID changes. (avijayan)
Posted by av...@apache.org.
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/ContainerHistoryData.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/ContainerHistoryData.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/ContainerHistoryData.java
deleted file mode 100644
index e606185..0000000
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/ContainerHistoryData.java
+++ /dev/null
@@ -1,182 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.server.applicationhistoryservice.records;
-
-import org.apache.hadoop.classification.InterfaceAudience.Public;
-import org.apache.hadoop.classification.InterfaceStability.Unstable;
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.api.records.ContainerState;
-import org.apache.hadoop.yarn.api.records.NodeId;
-import org.apache.hadoop.yarn.api.records.Priority;
-import org.apache.hadoop.yarn.api.records.Resource;
-
-/**
- * The class contains all the fields that are stored persistently for
- * <code>RMContainer</code>.
- */
-@Public
-@Unstable
-public class ContainerHistoryData {
-
- private ContainerId containerId;
-
- private Resource allocatedResource;
-
- private NodeId assignedNode;
-
- private Priority priority;
-
- private long startTime;
-
- private long finishTime;
-
- private String diagnosticsInfo;
-
- private int containerExitStatus;
-
- private ContainerState containerState;
-
- @Public
- @Unstable
- public static ContainerHistoryData newInstance(ContainerId containerId,
- Resource allocatedResource, NodeId assignedNode, Priority priority,
- long startTime, long finishTime, String diagnosticsInfo,
- int containerExitCode, ContainerState containerState) {
- ContainerHistoryData containerHD = new ContainerHistoryData();
- containerHD.setContainerId(containerId);
- containerHD.setAllocatedResource(allocatedResource);
- containerHD.setAssignedNode(assignedNode);
- containerHD.setPriority(priority);
- containerHD.setStartTime(startTime);
- containerHD.setFinishTime(finishTime);
- containerHD.setDiagnosticsInfo(diagnosticsInfo);
- containerHD.setContainerExitStatus(containerExitCode);
- containerHD.setContainerState(containerState);
- return containerHD;
- }
-
- @Public
- @Unstable
- public ContainerId getContainerId() {
- return containerId;
- }
-
- @Public
- @Unstable
- public void setContainerId(ContainerId containerId) {
- this.containerId = containerId;
- }
-
- @Public
- @Unstable
- public Resource getAllocatedResource() {
- return allocatedResource;
- }
-
- @Public
- @Unstable
- public void setAllocatedResource(Resource resource) {
- this.allocatedResource = resource;
- }
-
- @Public
- @Unstable
- public NodeId getAssignedNode() {
- return assignedNode;
- }
-
- @Public
- @Unstable
- public void setAssignedNode(NodeId nodeId) {
- this.assignedNode = nodeId;
- }
-
- @Public
- @Unstable
- public Priority getPriority() {
- return priority;
- }
-
- @Public
- @Unstable
- public void setPriority(Priority priority) {
- this.priority = priority;
- }
-
- @Public
- @Unstable
- public long getStartTime() {
- return startTime;
- }
-
- @Public
- @Unstable
- public void setStartTime(long startTime) {
- this.startTime = startTime;
- }
-
- @Public
- @Unstable
- public long getFinishTime() {
- return finishTime;
- }
-
- @Public
- @Unstable
- public void setFinishTime(long finishTime) {
- this.finishTime = finishTime;
- }
-
- @Public
- @Unstable
- public String getDiagnosticsInfo() {
- return diagnosticsInfo;
- }
-
- @Public
- @Unstable
- public void setDiagnosticsInfo(String diagnosticsInfo) {
- this.diagnosticsInfo = diagnosticsInfo;
- }
-
- @Public
- @Unstable
- public int getContainerExitStatus() {
- return containerExitStatus;
- }
-
- @Public
- @Unstable
- public void setContainerExitStatus(int containerExitStatus) {
- this.containerExitStatus = containerExitStatus;
- }
-
- @Public
- @Unstable
- public ContainerState getContainerState() {
- return containerState;
- }
-
- @Public
- @Unstable
- public void setContainerState(ContainerState containerState) {
- this.containerState = containerState;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/ContainerStartData.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/ContainerStartData.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/ContainerStartData.java
deleted file mode 100644
index 0c6dd81..0000000
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/ContainerStartData.java
+++ /dev/null
@@ -1,92 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.server.applicationhistoryservice.records;
-
-import org.apache.hadoop.classification.InterfaceAudience.Public;
-import org.apache.hadoop.classification.InterfaceStability.Unstable;
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.api.records.NodeId;
-import org.apache.hadoop.yarn.api.records.Priority;
-import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.util.Records;
-
-/**
- * The class contains the fields that can be determined when
- * <code>RMContainer</code> starts, and that need to be stored persistently.
- */
-@Public
-@Unstable
-public abstract class ContainerStartData {
-
- @Public
- @Unstable
- public static ContainerStartData newInstance(ContainerId containerId,
- Resource allocatedResource, NodeId assignedNode, Priority priority,
- long startTime) {
- ContainerStartData containerSD =
- Records.newRecord(ContainerStartData.class);
- containerSD.setContainerId(containerId);
- containerSD.setAllocatedResource(allocatedResource);
- containerSD.setAssignedNode(assignedNode);
- containerSD.setPriority(priority);
- containerSD.setStartTime(startTime);
- return containerSD;
- }
-
- @Public
- @Unstable
- public abstract ContainerId getContainerId();
-
- @Public
- @Unstable
- public abstract void setContainerId(ContainerId containerId);
-
- @Public
- @Unstable
- public abstract Resource getAllocatedResource();
-
- @Public
- @Unstable
- public abstract void setAllocatedResource(Resource resource);
-
- @Public
- @Unstable
- public abstract NodeId getAssignedNode();
-
- @Public
- @Unstable
- public abstract void setAssignedNode(NodeId nodeId);
-
- @Public
- @Unstable
- public abstract Priority getPriority();
-
- @Public
- @Unstable
- public abstract void setPriority(Priority priority);
-
- @Public
- @Unstable
- public abstract long getStartTime();
-
- @Public
- @Unstable
- public abstract void setStartTime(long startTime);
-
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/impl/pb/ApplicationAttemptFinishDataPBImpl.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/impl/pb/ApplicationAttemptFinishDataPBImpl.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/impl/pb/ApplicationAttemptFinishDataPBImpl.java
deleted file mode 100644
index 945c12f..0000000
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/impl/pb/ApplicationAttemptFinishDataPBImpl.java
+++ /dev/null
@@ -1,239 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.server.applicationhistoryservice.records.impl.pb;
-
-import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
-import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
-import org.apache.hadoop.yarn.api.records.YarnApplicationAttemptState;
-import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationAttemptIdPBImpl;
-import org.apache.hadoop.yarn.api.records.impl.pb.ProtoUtils;
-import org.apache.hadoop.yarn.proto.ApplicationHistoryServerProtos.ApplicationAttemptFinishDataProto;
-import org.apache.hadoop.yarn.proto.ApplicationHistoryServerProtos.ApplicationAttemptFinishDataProtoOrBuilder;
-import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationAttemptIdProto;
-import org.apache.hadoop.yarn.proto.YarnProtos.FinalApplicationStatusProto;
-import org.apache.hadoop.yarn.proto.YarnProtos.YarnApplicationAttemptStateProto;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ApplicationAttemptFinishData;
-
-import com.google.protobuf.TextFormat;
-
-public class ApplicationAttemptFinishDataPBImpl extends
- ApplicationAttemptFinishData {
-
- ApplicationAttemptFinishDataProto proto = ApplicationAttemptFinishDataProto
- .getDefaultInstance();
- ApplicationAttemptFinishDataProto.Builder builder = null;
- boolean viaProto = false;
-
- public ApplicationAttemptFinishDataPBImpl() {
- builder = ApplicationAttemptFinishDataProto.newBuilder();
- }
-
- public ApplicationAttemptFinishDataPBImpl(
- ApplicationAttemptFinishDataProto proto) {
- this.proto = proto;
- viaProto = true;
- }
-
- private ApplicationAttemptId applicationAttemptId;
-
- @Override
- public ApplicationAttemptId getApplicationAttemptId() {
- if (this.applicationAttemptId != null) {
- return this.applicationAttemptId;
- }
- ApplicationAttemptFinishDataProtoOrBuilder p = viaProto ? proto : builder;
- if (!p.hasApplicationAttemptId()) {
- return null;
- }
- this.applicationAttemptId =
- convertFromProtoFormat(p.getApplicationAttemptId());
- return this.applicationAttemptId;
- }
-
- @Override
- public void
- setApplicationAttemptId(ApplicationAttemptId applicationAttemptId) {
- maybeInitBuilder();
- if (applicationAttemptId == null) {
- builder.clearApplicationAttemptId();
- }
- this.applicationAttemptId = applicationAttemptId;
- }
-
- @Override
- public String getTrackingURL() {
- ApplicationAttemptFinishDataProtoOrBuilder p = viaProto ? proto : builder;
- if (!p.hasTrackingUrl()) {
- return null;
- }
- return p.getTrackingUrl();
- }
-
- @Override
- public void setTrackingURL(String trackingURL) {
- maybeInitBuilder();
- if (trackingURL == null) {
- builder.clearTrackingUrl();
- return;
- }
- builder.setTrackingUrl(trackingURL);
- }
-
- @Override
- public String getDiagnosticsInfo() {
- ApplicationAttemptFinishDataProtoOrBuilder p = viaProto ? proto : builder;
- if (!p.hasDiagnosticsInfo()) {
- return null;
- }
- return p.getDiagnosticsInfo();
- }
-
- @Override
- public void setDiagnosticsInfo(String diagnosticsInfo) {
- maybeInitBuilder();
- if (diagnosticsInfo == null) {
- builder.clearDiagnosticsInfo();
- return;
- }
- builder.setDiagnosticsInfo(diagnosticsInfo);
- }
-
- @Override
- public FinalApplicationStatus getFinalApplicationStatus() {
- ApplicationAttemptFinishDataProtoOrBuilder p = viaProto ? proto : builder;
- if (!p.hasFinalApplicationStatus()) {
- return null;
- }
- return convertFromProtoFormat(p.getFinalApplicationStatus());
- }
-
- @Override
- public void setFinalApplicationStatus(
- FinalApplicationStatus finalApplicationStatus) {
- maybeInitBuilder();
- if (finalApplicationStatus == null) {
- builder.clearFinalApplicationStatus();
- return;
- }
- builder
- .setFinalApplicationStatus(convertToProtoFormat(finalApplicationStatus));
- }
-
- @Override
- public YarnApplicationAttemptState getYarnApplicationAttemptState() {
- ApplicationAttemptFinishDataProtoOrBuilder p = viaProto ? proto : builder;
- if (!p.hasYarnApplicationAttemptState()) {
- return null;
- }
- return convertFromProtoFormat(p.getYarnApplicationAttemptState());
- }
-
- @Override
- public void setYarnApplicationAttemptState(YarnApplicationAttemptState state) {
- maybeInitBuilder();
- if (state == null) {
- builder.clearYarnApplicationAttemptState();
- return;
- }
- builder.setYarnApplicationAttemptState(convertToProtoFormat(state));
- }
-
- public ApplicationAttemptFinishDataProto getProto() {
- mergeLocalToProto();
- proto = viaProto ? proto : builder.build();
- viaProto = true;
- return proto;
- }
-
- @Override
- public int hashCode() {
- return getProto().hashCode();
- }
-
- @Override
- public boolean equals(Object other) {
- if (other == null)
- return false;
- if (other.getClass().isAssignableFrom(this.getClass())) {
- return this.getProto().equals(this.getClass().cast(other).getProto());
- }
- return false;
- }
-
- @Override
- public String toString() {
- return TextFormat.shortDebugString(getProto());
- }
-
- private void mergeLocalToBuilder() {
- if (this.applicationAttemptId != null
- && !((ApplicationAttemptIdPBImpl) this.applicationAttemptId).getProto()
- .equals(builder.getApplicationAttemptId())) {
- builder
- .setApplicationAttemptId(convertToProtoFormat(this.applicationAttemptId));
- }
- }
-
- private void mergeLocalToProto() {
- if (viaProto) {
- maybeInitBuilder();
- }
- mergeLocalToBuilder();
- proto = builder.build();
- viaProto = true;
- }
-
- private void maybeInitBuilder() {
- if (viaProto || builder == null) {
- builder = ApplicationAttemptFinishDataProto.newBuilder(proto);
- }
- viaProto = false;
- }
-
- private ApplicationAttemptIdPBImpl convertFromProtoFormat(
- ApplicationAttemptIdProto applicationAttemptId) {
- return new ApplicationAttemptIdPBImpl(applicationAttemptId);
- }
-
- private ApplicationAttemptIdProto convertToProtoFormat(
- ApplicationAttemptId applicationAttemptId) {
- return ((ApplicationAttemptIdPBImpl) applicationAttemptId).getProto();
- }
-
- private FinalApplicationStatus convertFromProtoFormat(
- FinalApplicationStatusProto finalApplicationStatus) {
- return ProtoUtils.convertFromProtoFormat(finalApplicationStatus);
- }
-
- private FinalApplicationStatusProto convertToProtoFormat(
- FinalApplicationStatus finalApplicationStatus) {
- return ProtoUtils.convertToProtoFormat(finalApplicationStatus);
- }
-
- private YarnApplicationAttemptStateProto convertToProtoFormat(
- YarnApplicationAttemptState state) {
- return ProtoUtils.convertToProtoFormat(state);
- }
-
- private YarnApplicationAttemptState convertFromProtoFormat(
- YarnApplicationAttemptStateProto yarnApplicationAttemptState) {
- return ProtoUtils.convertFromProtoFormat(yarnApplicationAttemptState);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/impl/pb/ApplicationAttemptStartDataPBImpl.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/impl/pb/ApplicationAttemptStartDataPBImpl.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/impl/pb/ApplicationAttemptStartDataPBImpl.java
deleted file mode 100644
index 1f67fc7..0000000
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/impl/pb/ApplicationAttemptStartDataPBImpl.java
+++ /dev/null
@@ -1,208 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.server.applicationhistoryservice.records.impl.pb;
-
-import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationAttemptIdPBImpl;
-import org.apache.hadoop.yarn.api.records.impl.pb.ContainerIdPBImpl;
-import org.apache.hadoop.yarn.proto.ApplicationHistoryServerProtos.ApplicationAttemptStartDataProto;
-import org.apache.hadoop.yarn.proto.ApplicationHistoryServerProtos.ApplicationAttemptStartDataProtoOrBuilder;
-import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationAttemptIdProto;
-import org.apache.hadoop.yarn.proto.YarnProtos.ContainerIdProto;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ApplicationAttemptStartData;
-
-import com.google.protobuf.TextFormat;
-
-public class ApplicationAttemptStartDataPBImpl extends
- ApplicationAttemptStartData {
-
- ApplicationAttemptStartDataProto proto = ApplicationAttemptStartDataProto
- .getDefaultInstance();
- ApplicationAttemptStartDataProto.Builder builder = null;
- boolean viaProto = false;
-
- public ApplicationAttemptStartDataPBImpl() {
- builder = ApplicationAttemptStartDataProto.newBuilder();
- }
-
- public ApplicationAttemptStartDataPBImpl(
- ApplicationAttemptStartDataProto proto) {
- this.proto = proto;
- viaProto = true;
- }
-
- private ApplicationAttemptId applicationAttemptId;
- private ContainerId masterContainerId;
-
- @Override
- public ApplicationAttemptId getApplicationAttemptId() {
- if (this.applicationAttemptId != null) {
- return this.applicationAttemptId;
- }
- ApplicationAttemptStartDataProtoOrBuilder p = viaProto ? proto : builder;
- if (!p.hasApplicationAttemptId()) {
- return null;
- }
- this.applicationAttemptId =
- convertFromProtoFormat(p.getApplicationAttemptId());
- return this.applicationAttemptId;
- }
-
- @Override
- public void
- setApplicationAttemptId(ApplicationAttemptId applicationAttemptId) {
- maybeInitBuilder();
- if (applicationAttemptId == null) {
- builder.clearApplicationAttemptId();
- }
- this.applicationAttemptId = applicationAttemptId;
- }
-
- @Override
- public String getHost() {
- ApplicationAttemptStartDataProtoOrBuilder p = viaProto ? proto : builder;
- if (!p.hasHost()) {
- return null;
- }
- return p.getHost();
- }
-
- @Override
- public void setHost(String host) {
- maybeInitBuilder();
- if (host == null) {
- builder.clearHost();
- return;
- }
- builder.setHost(host);
- }
-
- @Override
- public int getRPCPort() {
- ApplicationAttemptStartDataProtoOrBuilder p = viaProto ? proto : builder;
- return p.getRpcPort();
- }
-
- @Override
- public void setRPCPort(int rpcPort) {
- maybeInitBuilder();
- builder.setRpcPort(rpcPort);
- }
-
- @Override
- public ContainerId getMasterContainerId() {
- if (this.masterContainerId != null) {
- return this.masterContainerId;
- }
- ApplicationAttemptStartDataProtoOrBuilder p = viaProto ? proto : builder;
- if (!p.hasApplicationAttemptId()) {
- return null;
- }
- this.masterContainerId = convertFromProtoFormat(p.getMasterContainerId());
- return this.masterContainerId;
- }
-
- @Override
- public void setMasterContainerId(ContainerId masterContainerId) {
- maybeInitBuilder();
- if (masterContainerId == null) {
- builder.clearMasterContainerId();
- }
- this.masterContainerId = masterContainerId;
- }
-
- public ApplicationAttemptStartDataProto getProto() {
- mergeLocalToProto();
- proto = viaProto ? proto : builder.build();
- viaProto = true;
- return proto;
- }
-
- @Override
- public int hashCode() {
- return getProto().hashCode();
- }
-
- @Override
- public boolean equals(Object other) {
- if (other == null)
- return false;
- if (other.getClass().isAssignableFrom(this.getClass())) {
- return this.getProto().equals(this.getClass().cast(other).getProto());
- }
- return false;
- }
-
- @Override
- public String toString() {
- return TextFormat.shortDebugString(getProto());
- }
-
- private void mergeLocalToBuilder() {
- if (this.applicationAttemptId != null
- && !((ApplicationAttemptIdPBImpl) this.applicationAttemptId).getProto()
- .equals(builder.getApplicationAttemptId())) {
- builder
- .setApplicationAttemptId(convertToProtoFormat(this.applicationAttemptId));
- }
- if (this.masterContainerId != null
- && !((ContainerIdPBImpl) this.masterContainerId).getProto().equals(
- builder.getMasterContainerId())) {
- builder
- .setMasterContainerId(convertToProtoFormat(this.masterContainerId));
- }
- }
-
- private void mergeLocalToProto() {
- if (viaProto) {
- maybeInitBuilder();
- }
- mergeLocalToBuilder();
- proto = builder.build();
- viaProto = true;
- }
-
- private void maybeInitBuilder() {
- if (viaProto || builder == null) {
- builder = ApplicationAttemptStartDataProto.newBuilder(proto);
- }
- viaProto = false;
- }
-
- private ApplicationAttemptIdPBImpl convertFromProtoFormat(
- ApplicationAttemptIdProto applicationAttemptId) {
- return new ApplicationAttemptIdPBImpl(applicationAttemptId);
- }
-
- private ApplicationAttemptIdProto convertToProtoFormat(
- ApplicationAttemptId applicationAttemptId) {
- return ((ApplicationAttemptIdPBImpl) applicationAttemptId).getProto();
- }
-
- private ContainerIdPBImpl
- convertFromProtoFormat(ContainerIdProto containerId) {
- return new ContainerIdPBImpl(containerId);
- }
-
- private ContainerIdProto convertToProtoFormat(ContainerId masterContainerId) {
- return ((ContainerIdPBImpl) masterContainerId).getProto();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/impl/pb/ApplicationFinishDataPBImpl.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/impl/pb/ApplicationFinishDataPBImpl.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/impl/pb/ApplicationFinishDataPBImpl.java
deleted file mode 100644
index 337426d..0000000
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/impl/pb/ApplicationFinishDataPBImpl.java
+++ /dev/null
@@ -1,226 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.server.applicationhistoryservice.records.impl.pb;
-
-import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
-import org.apache.hadoop.yarn.api.records.YarnApplicationState;
-import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationIdPBImpl;
-import org.apache.hadoop.yarn.api.records.impl.pb.ProtoUtils;
-import org.apache.hadoop.yarn.proto.ApplicationHistoryServerProtos.ApplicationFinishDataProto;
-import org.apache.hadoop.yarn.proto.ApplicationHistoryServerProtos.ApplicationFinishDataProtoOrBuilder;
-import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationIdProto;
-import org.apache.hadoop.yarn.proto.YarnProtos.FinalApplicationStatusProto;
-import org.apache.hadoop.yarn.proto.YarnProtos.YarnApplicationStateProto;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ApplicationFinishData;
-
-import com.google.protobuf.TextFormat;
-
-public class ApplicationFinishDataPBImpl extends ApplicationFinishData {
-
- ApplicationFinishDataProto proto = ApplicationFinishDataProto
- .getDefaultInstance();
- ApplicationFinishDataProto.Builder builder = null;
- boolean viaProto = false;
-
- private ApplicationId applicationId;
-
- public ApplicationFinishDataPBImpl() {
- builder = ApplicationFinishDataProto.newBuilder();
- }
-
- public ApplicationFinishDataPBImpl(ApplicationFinishDataProto proto) {
- this.proto = proto;
- viaProto = true;
- }
-
- @Override
- public ApplicationId getApplicationId() {
- if (this.applicationId != null) {
- return this.applicationId;
- }
- ApplicationFinishDataProtoOrBuilder p = viaProto ? proto : builder;
- if (!p.hasApplicationId()) {
- return null;
- }
- this.applicationId = convertFromProtoFormat(p.getApplicationId());
- return this.applicationId;
- }
-
- @Override
- public void setApplicationId(ApplicationId applicationId) {
- maybeInitBuilder();
- if (applicationId == null) {
- builder.clearApplicationId();
- }
- this.applicationId = applicationId;
- }
-
- @Override
- public long getFinishTime() {
- ApplicationFinishDataProtoOrBuilder p = viaProto ? proto : builder;
- return p.getFinishTime();
- }
-
- @Override
- public void setFinishTime(long finishTime) {
- maybeInitBuilder();
- builder.setFinishTime(finishTime);
- }
-
- @Override
- public String getDiagnosticsInfo() {
- ApplicationFinishDataProtoOrBuilder p = viaProto ? proto : builder;
- if (!p.hasDiagnosticsInfo()) {
- return null;
- }
- return p.getDiagnosticsInfo();
- }
-
- @Override
- public void setDiagnosticsInfo(String diagnosticsInfo) {
- maybeInitBuilder();
- if (diagnosticsInfo == null) {
- builder.clearDiagnosticsInfo();
- return;
- }
- builder.setDiagnosticsInfo(diagnosticsInfo);
- }
-
- @Override
- public FinalApplicationStatus getFinalApplicationStatus() {
- ApplicationFinishDataProtoOrBuilder p = viaProto ? proto : builder;
- if (!p.hasFinalApplicationStatus()) {
- return null;
- }
- return convertFromProtoFormat(p.getFinalApplicationStatus());
- }
-
- @Override
- public void setFinalApplicationStatus(
- FinalApplicationStatus finalApplicationStatus) {
- maybeInitBuilder();
- if (finalApplicationStatus == null) {
- builder.clearFinalApplicationStatus();
- return;
- }
- builder
- .setFinalApplicationStatus(convertToProtoFormat(finalApplicationStatus));
- }
-
- @Override
- public YarnApplicationState getYarnApplicationState() {
- ApplicationFinishDataProtoOrBuilder p = viaProto ? proto : builder;
- if (!p.hasYarnApplicationState()) {
- return null;
- }
- return convertFromProtoFormat(p.getYarnApplicationState());
- }
-
- @Override
- public void setYarnApplicationState(YarnApplicationState state) {
- maybeInitBuilder();
- if (state == null) {
- builder.clearYarnApplicationState();
- return;
- }
- builder.setYarnApplicationState(convertToProtoFormat(state));
- }
-
- public ApplicationFinishDataProto getProto() {
- mergeLocalToProto();
- proto = viaProto ? proto : builder.build();
- viaProto = true;
- return proto;
- }
-
- @Override
- public int hashCode() {
- return getProto().hashCode();
- }
-
- @Override
- public boolean equals(Object other) {
- if (other == null)
- return false;
- if (other.getClass().isAssignableFrom(this.getClass())) {
- return this.getProto().equals(this.getClass().cast(other).getProto());
- }
- return false;
- }
-
- @Override
- public String toString() {
- return TextFormat.shortDebugString(getProto());
- }
-
- private void mergeLocalToBuilder() {
- if (this.applicationId != null
- && !((ApplicationIdPBImpl) this.applicationId).getProto().equals(
- builder.getApplicationId())) {
- builder.setApplicationId(convertToProtoFormat(this.applicationId));
- }
- }
-
- private void mergeLocalToProto() {
- if (viaProto) {
- maybeInitBuilder();
- }
- mergeLocalToBuilder();
- proto = builder.build();
- viaProto = true;
- }
-
- private void maybeInitBuilder() {
- if (viaProto || builder == null) {
- builder = ApplicationFinishDataProto.newBuilder(proto);
- }
- viaProto = false;
- }
-
- private ApplicationIdProto convertToProtoFormat(ApplicationId applicationId) {
- return ((ApplicationIdPBImpl) applicationId).getProto();
- }
-
- private ApplicationIdPBImpl convertFromProtoFormat(
- ApplicationIdProto applicationId) {
- return new ApplicationIdPBImpl(applicationId);
- }
-
- private FinalApplicationStatus convertFromProtoFormat(
- FinalApplicationStatusProto finalApplicationStatus) {
- return ProtoUtils.convertFromProtoFormat(finalApplicationStatus);
- }
-
- private FinalApplicationStatusProto convertToProtoFormat(
- FinalApplicationStatus finalApplicationStatus) {
- return ProtoUtils.convertToProtoFormat(finalApplicationStatus);
- }
-
- private YarnApplicationStateProto convertToProtoFormat(
- YarnApplicationState state) {
- return ProtoUtils.convertToProtoFormat(state);
- }
-
- private YarnApplicationState convertFromProtoFormat(
- YarnApplicationStateProto yarnApplicationState) {
- return ProtoUtils.convertFromProtoFormat(yarnApplicationState);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/impl/pb/ApplicationStartDataPBImpl.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/impl/pb/ApplicationStartDataPBImpl.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/impl/pb/ApplicationStartDataPBImpl.java
deleted file mode 100644
index 56f7aff..0000000
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/impl/pb/ApplicationStartDataPBImpl.java
+++ /dev/null
@@ -1,229 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.server.applicationhistoryservice.records.impl.pb;
-
-import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationIdPBImpl;
-import org.apache.hadoop.yarn.proto.ApplicationHistoryServerProtos.ApplicationStartDataProto;
-import org.apache.hadoop.yarn.proto.ApplicationHistoryServerProtos.ApplicationStartDataProtoOrBuilder;
-import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationIdProto;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ApplicationStartData;
-
-import com.google.protobuf.TextFormat;
-
-public class ApplicationStartDataPBImpl extends ApplicationStartData {
-
- ApplicationStartDataProto proto = ApplicationStartDataProto
- .getDefaultInstance();
- ApplicationStartDataProto.Builder builder = null;
- boolean viaProto = false;
-
- private ApplicationId applicationId;
-
- public ApplicationStartDataPBImpl() {
- builder = ApplicationStartDataProto.newBuilder();
- }
-
- public ApplicationStartDataPBImpl(ApplicationStartDataProto proto) {
- this.proto = proto;
- viaProto = true;
- }
-
- @Override
- public ApplicationId getApplicationId() {
- if (this.applicationId != null) {
- return this.applicationId;
- }
- ApplicationStartDataProtoOrBuilder p = viaProto ? proto : builder;
- if (!p.hasApplicationId()) {
- return null;
- }
- this.applicationId = convertFromProtoFormat(p.getApplicationId());
- return this.applicationId;
- }
-
- @Override
- public void setApplicationId(ApplicationId applicationId) {
- maybeInitBuilder();
- if (applicationId == null) {
- builder.clearApplicationId();
- }
- this.applicationId = applicationId;
- }
-
- @Override
- public String getApplicationName() {
- ApplicationStartDataProtoOrBuilder p = viaProto ? proto : builder;
- if (!p.hasApplicationName()) {
- return null;
- }
- return p.getApplicationName();
- }
-
- @Override
- public void setApplicationName(String applicationName) {
- maybeInitBuilder();
- if (applicationName == null) {
- builder.clearApplicationName();
- return;
- }
- builder.setApplicationName(applicationName);
- }
-
- @Override
- public String getApplicationType() {
- ApplicationStartDataProtoOrBuilder p = viaProto ? proto : builder;
- if (!p.hasApplicationType()) {
- return null;
- }
- return p.getApplicationType();
- }
-
- @Override
- public void setApplicationType(String applicationType) {
- maybeInitBuilder();
- if (applicationType == null) {
- builder.clearApplicationType();
- return;
- }
- builder.setApplicationType(applicationType);
- }
-
- @Override
- public String getUser() {
- ApplicationStartDataProtoOrBuilder p = viaProto ? proto : builder;
- if (!p.hasUser()) {
- return null;
- }
- return p.getUser();
- }
-
- @Override
- public void setUser(String user) {
- maybeInitBuilder();
- if (user == null) {
- builder.clearUser();
- return;
- }
- builder.setUser(user);
- }
-
- @Override
- public String getQueue() {
- ApplicationStartDataProtoOrBuilder p = viaProto ? proto : builder;
- if (!p.hasQueue()) {
- return null;
- }
- return p.getQueue();
- }
-
- @Override
- public void setQueue(String queue) {
- maybeInitBuilder();
- if (queue == null) {
- builder.clearQueue();
- return;
- }
- builder.setQueue(queue);
- }
-
- @Override
- public long getSubmitTime() {
- ApplicationStartDataProtoOrBuilder p = viaProto ? proto : builder;
- return p.getSubmitTime();
- }
-
- @Override
- public void setSubmitTime(long submitTime) {
- maybeInitBuilder();
- builder.setSubmitTime(submitTime);
- }
-
- @Override
- public long getStartTime() {
- ApplicationStartDataProtoOrBuilder p = viaProto ? proto : builder;
- return p.getStartTime();
- }
-
- @Override
- public void setStartTime(long startTime) {
- maybeInitBuilder();
- builder.setStartTime(startTime);
- }
-
- public ApplicationStartDataProto getProto() {
- mergeLocalToProto();
- proto = viaProto ? proto : builder.build();
- viaProto = true;
- return proto;
- }
-
- @Override
- public int hashCode() {
- return getProto().hashCode();
- }
-
- @Override
- public boolean equals(Object other) {
- if (other == null)
- return false;
- if (other.getClass().isAssignableFrom(this.getClass())) {
- return this.getProto().equals(this.getClass().cast(other).getProto());
- }
- return false;
- }
-
- @Override
- public String toString() {
- return TextFormat.shortDebugString(getProto());
- }
-
- private void mergeLocalToBuilder() {
- if (this.applicationId != null
- && !((ApplicationIdPBImpl) this.applicationId).getProto().equals(
- builder.getApplicationId())) {
- builder.setApplicationId(convertToProtoFormat(this.applicationId));
- }
- }
-
- private void mergeLocalToProto() {
- if (viaProto) {
- maybeInitBuilder();
- }
- mergeLocalToBuilder();
- proto = builder.build();
- viaProto = true;
- }
-
- private void maybeInitBuilder() {
- if (viaProto || builder == null) {
- builder = ApplicationStartDataProto.newBuilder(proto);
- }
- viaProto = false;
- }
-
- private ApplicationIdProto convertToProtoFormat(ApplicationId applicationId) {
- return ((ApplicationIdPBImpl) applicationId).getProto();
- }
-
- private ApplicationIdPBImpl convertFromProtoFormat(
- ApplicationIdProto applicationId) {
- return new ApplicationIdPBImpl(applicationId);
- }
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/impl/pb/ContainerFinishDataPBImpl.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/impl/pb/ContainerFinishDataPBImpl.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/impl/pb/ContainerFinishDataPBImpl.java
deleted file mode 100644
index 8bc01e0..0000000
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/impl/pb/ContainerFinishDataPBImpl.java
+++ /dev/null
@@ -1,204 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.server.applicationhistoryservice.records.impl.pb;
-
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.api.records.ContainerState;
-import org.apache.hadoop.yarn.api.records.impl.pb.ContainerIdPBImpl;
-import org.apache.hadoop.yarn.api.records.impl.pb.ProtoUtils;
-import org.apache.hadoop.yarn.proto.ApplicationHistoryServerProtos.ContainerFinishDataProto;
-import org.apache.hadoop.yarn.proto.ApplicationHistoryServerProtos.ContainerFinishDataProtoOrBuilder;
-import org.apache.hadoop.yarn.proto.YarnProtos.ContainerIdProto;
-import org.apache.hadoop.yarn.proto.YarnProtos.ContainerStateProto;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ContainerFinishData;
-
-import com.google.protobuf.TextFormat;
-
-public class ContainerFinishDataPBImpl extends ContainerFinishData {
-
- ContainerFinishDataProto proto = ContainerFinishDataProto
- .getDefaultInstance();
- ContainerFinishDataProto.Builder builder = null;
- boolean viaProto = false;
-
- private ContainerId containerId;
-
- public ContainerFinishDataPBImpl() {
- builder = ContainerFinishDataProto.newBuilder();
- }
-
- public ContainerFinishDataPBImpl(ContainerFinishDataProto proto) {
- this.proto = proto;
- viaProto = true;
- }
-
- @Override
- public ContainerId getContainerId() {
- if (this.containerId != null) {
- return this.containerId;
- }
- ContainerFinishDataProtoOrBuilder p = viaProto ? proto : builder;
- if (!p.hasContainerId()) {
- return null;
- }
- this.containerId = convertFromProtoFormat(p.getContainerId());
- return this.containerId;
- }
-
- @Override
- public void setContainerId(ContainerId containerId) {
- maybeInitBuilder();
- if (containerId == null) {
- builder.clearContainerId();
- }
- this.containerId = containerId;
- }
-
- @Override
- public long getFinishTime() {
- ContainerFinishDataProtoOrBuilder p = viaProto ? proto : builder;
- return p.getFinishTime();
- }
-
- @Override
- public void setFinishTime(long finishTime) {
- maybeInitBuilder();
- builder.setFinishTime(finishTime);
- }
-
- @Override
- public String getDiagnosticsInfo() {
- ContainerFinishDataProtoOrBuilder p = viaProto ? proto : builder;
- if (!p.hasDiagnosticsInfo()) {
- return null;
- }
- return p.getDiagnosticsInfo();
- }
-
- @Override
- public void setDiagnosticsInfo(String diagnosticsInfo) {
- maybeInitBuilder();
- if (diagnosticsInfo == null) {
- builder.clearDiagnosticsInfo();
- return;
- }
- builder.setDiagnosticsInfo(diagnosticsInfo);
- }
-
- @Override
- public int getContainerExitStatus() {
- ContainerFinishDataProtoOrBuilder p = viaProto ? proto : builder;
- return p.getContainerExitStatus();
- }
-
- @Override
- public ContainerState getContainerState() {
- ContainerFinishDataProtoOrBuilder p = viaProto ? proto : builder;
- if (!p.hasContainerState()) {
- return null;
- }
- return convertFromProtoFormat(p.getContainerState());
- }
-
- @Override
- public void setContainerState(ContainerState state) {
- maybeInitBuilder();
- if (state == null) {
- builder.clearContainerState();
- return;
- }
- builder.setContainerState(convertToProtoFormat(state));
- }
-
- @Override
- public void setContainerExitStatus(int containerExitStatus) {
- maybeInitBuilder();
- builder.setContainerExitStatus(containerExitStatus);
- }
-
- public ContainerFinishDataProto getProto() {
- mergeLocalToProto();
- proto = viaProto ? proto : builder.build();
- viaProto = true;
- return proto;
- }
-
- @Override
- public int hashCode() {
- return getProto().hashCode();
- }
-
- @Override
- public boolean equals(Object other) {
- if (other == null)
- return false;
- if (other.getClass().isAssignableFrom(this.getClass())) {
- return this.getProto().equals(this.getClass().cast(other).getProto());
- }
- return false;
- }
-
- @Override
- public String toString() {
- return TextFormat.shortDebugString(getProto());
- }
-
- private void mergeLocalToBuilder() {
- if (this.containerId != null
- && !((ContainerIdPBImpl) this.containerId).getProto().equals(
- builder.getContainerId())) {
- builder.setContainerId(convertToProtoFormat(this.containerId));
- }
- }
-
- private void mergeLocalToProto() {
- if (viaProto) {
- maybeInitBuilder();
- }
- mergeLocalToBuilder();
- proto = builder.build();
- viaProto = true;
- }
-
- private void maybeInitBuilder() {
- if (viaProto || builder == null) {
- builder = ContainerFinishDataProto.newBuilder(proto);
- }
- viaProto = false;
- }
-
- private ContainerIdProto convertToProtoFormat(ContainerId containerId) {
- return ((ContainerIdPBImpl) containerId).getProto();
- }
-
- private ContainerIdPBImpl
- convertFromProtoFormat(ContainerIdProto containerId) {
- return new ContainerIdPBImpl(containerId);
- }
-
- private ContainerStateProto convertToProtoFormat(ContainerState state) {
- return ProtoUtils.convertToProtoFormat(state);
- }
-
- private ContainerState convertFromProtoFormat(
- ContainerStateProto containerState) {
- return ProtoUtils.convertFromProtoFormat(containerState);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/impl/pb/ContainerStartDataPBImpl.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/impl/pb/ContainerStartDataPBImpl.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/impl/pb/ContainerStartDataPBImpl.java
deleted file mode 100644
index 6d248b2..0000000
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/impl/pb/ContainerStartDataPBImpl.java
+++ /dev/null
@@ -1,258 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.server.applicationhistoryservice.records.impl.pb;
-
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.api.records.NodeId;
-import org.apache.hadoop.yarn.api.records.Priority;
-import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.api.records.impl.pb.ContainerIdPBImpl;
-import org.apache.hadoop.yarn.api.records.impl.pb.NodeIdPBImpl;
-import org.apache.hadoop.yarn.api.records.impl.pb.PriorityPBImpl;
-import org.apache.hadoop.yarn.api.records.impl.pb.ResourcePBImpl;
-import org.apache.hadoop.yarn.proto.ApplicationHistoryServerProtos.ContainerStartDataProto;
-import org.apache.hadoop.yarn.proto.ApplicationHistoryServerProtos.ContainerStartDataProtoOrBuilder;
-import org.apache.hadoop.yarn.proto.YarnProtos.ContainerIdProto;
-import org.apache.hadoop.yarn.proto.YarnProtos.NodeIdProto;
-import org.apache.hadoop.yarn.proto.YarnProtos.PriorityProto;
-import org.apache.hadoop.yarn.proto.YarnProtos.ResourceProto;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ContainerStartData;
-
-import com.google.protobuf.TextFormat;
-
-public class ContainerStartDataPBImpl extends ContainerStartData {
-
- ContainerStartDataProto proto = ContainerStartDataProto.getDefaultInstance();
- ContainerStartDataProto.Builder builder = null;
- boolean viaProto = false;
-
- private ContainerId containerId;
- private Resource resource;
- private NodeId nodeId;
- private Priority priority;
-
- public ContainerStartDataPBImpl() {
- builder = ContainerStartDataProto.newBuilder();
- }
-
- public ContainerStartDataPBImpl(ContainerStartDataProto proto) {
- this.proto = proto;
- viaProto = true;
- }
-
- @Override
- public ContainerId getContainerId() {
- if (this.containerId != null) {
- return this.containerId;
- }
- ContainerStartDataProtoOrBuilder p = viaProto ? proto : builder;
- if (!p.hasContainerId()) {
- return null;
- }
- this.containerId = convertFromProtoFormat(p.getContainerId());
- return this.containerId;
- }
-
- @Override
- public void setContainerId(ContainerId containerId) {
- maybeInitBuilder();
- if (containerId == null) {
- builder.clearContainerId();
- }
- this.containerId = containerId;
- }
-
- @Override
- public Resource getAllocatedResource() {
- if (this.resource != null) {
- return this.resource;
- }
- ContainerStartDataProtoOrBuilder p = viaProto ? proto : builder;
- if (!p.hasAllocatedResource()) {
- return null;
- }
- this.resource = convertFromProtoFormat(p.getAllocatedResource());
- return this.resource;
- }
-
- @Override
- public void setAllocatedResource(Resource resource) {
- maybeInitBuilder();
- if (resource == null) {
- builder.clearAllocatedResource();
- }
- this.resource = resource;
- }
-
- @Override
- public NodeId getAssignedNode() {
- if (this.nodeId != null) {
- return this.nodeId;
- }
- ContainerStartDataProtoOrBuilder p = viaProto ? proto : builder;
- if (!p.hasAssignedNodeId()) {
- return null;
- }
- this.nodeId = convertFromProtoFormat(p.getAssignedNodeId());
- return this.nodeId;
- }
-
- @Override
- public void setAssignedNode(NodeId nodeId) {
- maybeInitBuilder();
- if (nodeId == null) {
- builder.clearAssignedNodeId();
- }
- this.nodeId = nodeId;
- }
-
- @Override
- public Priority getPriority() {
- if (this.priority != null) {
- return this.priority;
- }
- ContainerStartDataProtoOrBuilder p = viaProto ? proto : builder;
- if (!p.hasPriority()) {
- return null;
- }
- this.priority = convertFromProtoFormat(p.getPriority());
- return this.priority;
- }
-
- @Override
- public void setPriority(Priority priority) {
- maybeInitBuilder();
- if (priority == null) {
- builder.clearPriority();
- }
- this.priority = priority;
- }
-
- @Override
- public long getStartTime() {
- ContainerStartDataProtoOrBuilder p = viaProto ? proto : builder;
- return p.getStartTime();
- }
-
- @Override
- public void setStartTime(long startTime) {
- maybeInitBuilder();
- builder.setStartTime(startTime);
- }
-
- public ContainerStartDataProto getProto() {
- mergeLocalToProto();
- proto = viaProto ? proto : builder.build();
- viaProto = true;
- return proto;
- }
-
- @Override
- public int hashCode() {
- return getProto().hashCode();
- }
-
- @Override
- public boolean equals(Object other) {
- if (other == null)
- return false;
- if (other.getClass().isAssignableFrom(this.getClass())) {
- return this.getProto().equals(this.getClass().cast(other).getProto());
- }
- return false;
- }
-
- @Override
- public String toString() {
- return TextFormat.shortDebugString(getProto());
- }
-
- private void mergeLocalToBuilder() {
- if (this.containerId != null
- && !((ContainerIdPBImpl) this.containerId).getProto().equals(
- builder.getContainerId())) {
- builder.setContainerId(convertToProtoFormat(this.containerId));
- }
- if (this.resource != null
- && !((ResourcePBImpl) this.resource).getProto().equals(
- builder.getAllocatedResource())) {
- builder.setAllocatedResource(convertToProtoFormat(this.resource));
- }
- if (this.nodeId != null
- && !((NodeIdPBImpl) this.nodeId).getProto().equals(
- builder.getAssignedNodeId())) {
- builder.setAssignedNodeId(convertToProtoFormat(this.nodeId));
- }
- if (this.priority != null
- && !((PriorityPBImpl) this.priority).getProto().equals(
- builder.getPriority())) {
- builder.setPriority(convertToProtoFormat(this.priority));
- }
- }
-
- private void mergeLocalToProto() {
- if (viaProto) {
- maybeInitBuilder();
- }
- mergeLocalToBuilder();
- proto = builder.build();
- viaProto = true;
- }
-
- private void maybeInitBuilder() {
- if (viaProto || builder == null) {
- builder = ContainerStartDataProto.newBuilder(proto);
- }
- viaProto = false;
- }
-
- private ContainerIdProto convertToProtoFormat(ContainerId containerId) {
- return ((ContainerIdPBImpl) containerId).getProto();
- }
-
- private ContainerIdPBImpl
- convertFromProtoFormat(ContainerIdProto containerId) {
- return new ContainerIdPBImpl(containerId);
- }
-
- private ResourceProto convertToProtoFormat(Resource resource) {
- return ((ResourcePBImpl) resource).getProto();
- }
-
- private ResourcePBImpl convertFromProtoFormat(ResourceProto resource) {
- return new ResourcePBImpl(resource);
- }
-
- private NodeIdProto convertToProtoFormat(NodeId nodeId) {
- return ((NodeIdPBImpl) nodeId).getProto();
- }
-
- private NodeIdPBImpl convertFromProtoFormat(NodeIdProto nodeId) {
- return new NodeIdPBImpl(nodeId);
- }
-
- private PriorityProto convertToProtoFormat(Priority priority) {
- return ((PriorityPBImpl) priority).getProto();
- }
-
- private PriorityPBImpl convertFromProtoFormat(PriorityProto priority) {
- return new PriorityPBImpl(priority);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/GenericObjectMapper.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/GenericObjectMapper.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/GenericObjectMapper.java
deleted file mode 100644
index b1846a3..0000000
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/GenericObjectMapper.java
+++ /dev/null
@@ -1,135 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.applicationhistoryservice.timeline;
-
-import java.io.IOException;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-import org.codehaus.jackson.map.ObjectMapper;
-import org.codehaus.jackson.map.ObjectReader;
-import org.codehaus.jackson.map.ObjectWriter;
-
-/**
- * A utility class providing methods for serializing and deserializing
- * objects. The {@link #write(Object)} and {@link #read(byte[])} methods are
- * used by the {@link LeveldbTimelineStore} to store and retrieve arbitrary
- * JSON, while the {@link #writeReverseOrderedLong} and {@link
- * #readReverseOrderedLong} methods are used to sort entities in descending
- * start time order.
- */
-@InterfaceAudience.Private
-@InterfaceStability.Unstable
-public class GenericObjectMapper {
- private static final byte[] EMPTY_BYTES = new byte[0];
-
- public static final ObjectReader OBJECT_READER;
- public static final ObjectWriter OBJECT_WRITER;
-
- static {
- ObjectMapper mapper = new ObjectMapper();
- OBJECT_READER = mapper.reader(Object.class);
- OBJECT_WRITER = mapper.writer();
- }
-
- /**
- * Serializes an Object into a byte array. Along with {@link #read(byte[])},
- * can be used to serialize an Object and deserialize it into an Object of
- * the same type without needing to specify the Object's type,
- * as long as it is one of the JSON-compatible objects understood by
- * ObjectMapper.
- *
- * @param o An Object
- * @return A byte array representation of the Object
- * @throws IOException if there is a write error
- */
- public static byte[] write(Object o) throws IOException {
- if (o == null) {
- return EMPTY_BYTES;
- }
- return OBJECT_WRITER.writeValueAsBytes(o);
- }
-
- /**
- * Deserializes an Object from a byte array created with
- * {@link #write(Object)}.
- *
- * @param b A byte array
- * @return An Object
- * @throws IOException if there is a read error
- */
- public static Object read(byte[] b) throws IOException {
- return read(b, 0);
- }
-
- /**
- * Deserializes an Object from a byte array at a specified offset, assuming
- * the bytes were created with {@link #write(Object)}.
- *
- * @param b A byte array
- * @param offset Offset into the array
- * @return An Object
- * @throws IOException if there is a read error
- */
- public static Object read(byte[] b, int offset) throws IOException {
- if (b == null || b.length == 0) {
- return null;
- }
- return OBJECT_READER.readValue(b, offset, b.length - offset);
- }
-
- /**
- * Converts a long to a 8-byte array so that lexicographic ordering of the
- * produced byte arrays sort the longs in descending order.
- *
- * @param l A long
- * @return A byte array
- */
- public static byte[] writeReverseOrderedLong(long l) {
- byte[] b = new byte[8];
- return writeReverseOrderedLong(l, b, 0);
- }
-
- public static byte[] writeReverseOrderedLong(long l, byte[] b, int offset) {
- b[offset] = (byte)(0x7f ^ ((l >> 56) & 0xff));
- for (int i = offset+1; i < offset+7; i++) {
- b[i] = (byte)(0xff ^ ((l >> 8*(7-i)) & 0xff));
- }
- b[offset+7] = (byte)(0xff ^ (l & 0xff));
- return b;
- }
-
- /**
- * Reads 8 bytes from an array starting at the specified offset and
- * converts them to a long. The bytes are assumed to have been created
- * with {@link #writeReverseOrderedLong}.
- *
- * @param b A byte array
- * @param offset An offset into the byte array
- * @return A long
- */
- public static long readReverseOrderedLong(byte[] b, int offset) {
- long l = b[offset] & 0xff;
- for (int i = 1; i < 8; i++) {
- l = l << 8;
- l = l | (b[offset+i]&0xff);
- }
- return l ^ 0x7fffffffffffffffl;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/NameValuePair.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/NameValuePair.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/NameValuePair.java
deleted file mode 100644
index d8dabd2..0000000
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/NameValuePair.java
+++ /dev/null
@@ -1,59 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.applicationhistoryservice.timeline;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-
-/**
- * A class holding a name and value pair, used for specifying filters in
- * {@link TimelineReader}.
- */
-@InterfaceAudience.Private
-@InterfaceStability.Unstable
-public class NameValuePair {
- String name;
- Object value;
-
- public NameValuePair(String name, Object value) {
- this.name = name;
- this.value = value;
- }
-
- /**
- * Get the name.
- * @return The name.
- */
- public String getName() {
-
- return name;
- }
-
- /**
- * Get the value.
- * @return The value.
- */
- public Object getValue() {
- return value;
- }
-
- @Override
- public String toString() {
- return "{ name: " + name + ", value: " + value + " }";
- }
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/TimelineReader.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/TimelineReader.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/TimelineReader.java
deleted file mode 100644
index 9ae9954..0000000
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/TimelineReader.java
+++ /dev/null
@@ -1,155 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.server.applicationhistoryservice.timeline;
-
-import java.io.IOException;
-import java.util.Collection;
-import java.util.EnumSet;
-import java.util.Set;
-import java.util.SortedSet;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.yarn.api.records.timeline.TimelineEntities;
-import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity;
-import org.apache.hadoop.yarn.api.records.timeline.TimelineEvents;
-
-/**
- * This interface is for retrieving timeline information.
- */
-@InterfaceAudience.Private
-@InterfaceStability.Unstable
-public interface TimelineReader {
-
- /**
- * Possible fields to retrieve for {@link #getEntities} and {@link #getEntity}
- * .
- */
- enum Field {
- EVENTS,
- RELATED_ENTITIES,
- PRIMARY_FILTERS,
- OTHER_INFO,
- LAST_EVENT_ONLY
- }
-
- /**
- * Default limit for {@link #getEntities} and {@link #getEntityTimelines}.
- */
- final long DEFAULT_LIMIT = 100;
-
- /**
- * This method retrieves a list of entity information, {@link TimelineEntity},
- * sorted by the starting timestamp for the entity, descending. The starting
- * timestamp of an entity is a timestamp specified by the client. If it is not
- * explicitly specified, it will be chosen by the store to be the earliest
- * timestamp of the events received in the first put for the entity.
- *
- * @param entityType
- * The type of entities to return (required).
- * @param limit
- * A limit on the number of entities to return. If null, defaults to
- * {@link #DEFAULT_LIMIT}.
- * @param windowStart
- * The earliest start timestamp to retrieve (exclusive). If null,
- * defaults to retrieving all entities until the limit is reached.
- * @param windowEnd
- * The latest start timestamp to retrieve (inclusive). If null,
- * defaults to {@link Long#MAX_VALUE}
- * @param fromId
- * If fromId is not null, retrieve entities earlier than and
- * including the specified ID. If no start time is found for the
- * specified ID, an empty list of entities will be returned. The
- * windowEnd parameter will take precedence if the start time of this
- * entity falls later than windowEnd.
- * @param fromTs
- * If fromTs is not null, ignore entities that were inserted into the
- * store after the given timestamp. The entity's insert timestamp
- * used for this comparison is the store's system time when the first
- * put for the entity was received (not the entity's start time).
- * @param primaryFilter
- * Retrieves only entities that have the specified primary filter. If
- * null, retrieves all entities. This is an indexed retrieval, and no
- * entities that do not match the filter are scanned.
- * @param secondaryFilters
- * Retrieves only entities that have exact matches for all the
- * specified filters in their primary filters or other info. This is
- * not an indexed retrieval, so all entities are scanned but only
- * those matching the filters are returned.
- * @param fieldsToRetrieve
- * Specifies which fields of the entity object to retrieve (see
- * {@link Field}). If the set of fields contains
- * {@link Field#LAST_EVENT_ONLY} and not {@link Field#EVENTS}, the
- * most recent event for each entity is retrieved. If null, retrieves
- * all fields.
- * @return An {@link TimelineEntities} object.
- * @throws IOException
- */
- TimelineEntities getEntities(String entityType,
- Long limit, Long windowStart, Long windowEnd, String fromId, Long fromTs,
- NameValuePair primaryFilter, Collection<NameValuePair> secondaryFilters,
- EnumSet<Field> fieldsToRetrieve) throws IOException;
-
- /**
- * This method retrieves the entity information for a given entity.
- *
- * @param entityId
- * The entity whose information will be retrieved.
- * @param entityType
- * The type of the entity.
- * @param fieldsToRetrieve
- * Specifies which fields of the entity object to retrieve (see
- * {@link Field}). If the set of fields contains
- * {@link Field#LAST_EVENT_ONLY} and not {@link Field#EVENTS}, the
- * most recent event for each entity is retrieved. If null, retrieves
- * all fields.
- * @return An {@link TimelineEntity} object.
- * @throws IOException
- */
- TimelineEntity getEntity(String entityId, String entityType, EnumSet<Field>
- fieldsToRetrieve) throws IOException;
-
- /**
- * This method retrieves the events for a list of entities all of the same
- * entity type. The events for each entity are sorted in order of their
- * timestamps, descending.
- *
- * @param entityType
- * The type of entities to retrieve events for.
- * @param entityIds
- * The entity IDs to retrieve events for.
- * @param limit
- * A limit on the number of events to return for each entity. If
- * null, defaults to {@link #DEFAULT_LIMIT} events per entity.
- * @param windowStart
- * If not null, retrieves only events later than the given time
- * (exclusive)
- * @param windowEnd
- * If not null, retrieves only events earlier than the given time
- * (inclusive)
- * @param eventTypes
- * Restricts the events returned to the given types. If null, events
- * of all types will be returned.
- * @return An {@link TimelineEvents} object.
- * @throws IOException
- */
- TimelineEvents getEntityTimelines(String entityType,
- SortedSet<String> entityIds, Long limit, Long windowStart,
- Long windowEnd, Set<String> eventTypes) throws IOException;
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/TimelineStore.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/TimelineStore.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/TimelineStore.java
deleted file mode 100644
index 6b50d83..0000000
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/TimelineStore.java
+++ /dev/null
@@ -1,29 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.server.applicationhistoryservice.timeline;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.service.Service;
-
-@InterfaceAudience.Private
-@InterfaceStability.Unstable
-public interface TimelineStore extends
- Service, TimelineReader, TimelineWriter {
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/TimelineWriter.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/TimelineWriter.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/TimelineWriter.java
deleted file mode 100644
index bc8aada..0000000
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/TimelineWriter.java
+++ /dev/null
@@ -1,46 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.server.applicationhistoryservice.timeline;
-
-import java.io.IOException;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.yarn.api.records.timeline.TimelineEntities;
-import org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse;
-
-/**
- * This interface is for storing timeline information.
- */
-@InterfaceAudience.Private
-@InterfaceStability.Unstable
-public interface TimelineWriter {
-
- /**
- * Stores entity information to the timeline store. Any errors occurring for
- * individual put request objects will be reported in the response.
- *
- * @param data
- * An {@link TimelineEntities} object.
- * @return An {@link TimelinePutResponse} object.
- * @throws IOException
- */
- TimelinePutResponse put(TimelineEntities data) throws IOException;
-
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AMSController.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AMSController.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AMSController.java
deleted file mode 100644
index 0bf962e..0000000
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AMSController.java
+++ /dev/null
@@ -1,37 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.server.applicationhistoryservice.webapp;
-
-import org.apache.hadoop.yarn.webapp.Controller;
-
-import com.google.inject.Inject;
-
-public class AMSController extends Controller {
-
- @Inject
- AMSController(RequestContext ctx) {
- super(ctx);
- }
-
- @Override
- public void index() {
- setTitle("Ambari Metrics Service");
- }
-
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AMSWebApp.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AMSWebApp.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AMSWebApp.java
deleted file mode 100644
index 2f6eec7..0000000
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AMSWebApp.java
+++ /dev/null
@@ -1,42 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.applicationhistoryservice.webapp;
-
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricStore;
-import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
-import org.apache.hadoop.yarn.webapp.WebApp;
-import org.apache.hadoop.yarn.webapp.YarnJacksonJaxbJsonProvider;
-import org.apache.hadoop.yarn.webapp.YarnWebParams;
-
-public class AMSWebApp extends WebApp implements YarnWebParams {
-
- private final TimelineMetricStore timelineMetricStore;
-
- public AMSWebApp(TimelineMetricStore timelineMetricStore) {
- this.timelineMetricStore = timelineMetricStore;
- }
-
- @Override
- public void setup() {
- bind(YarnJacksonJaxbJsonProvider.class);
- bind(TimelineWebServices.class);
- bind(GenericExceptionHandler.class);
- bind(TimelineMetricStore.class).toInstance(timelineMetricStore);
- route("/", AMSController.class);
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/JAXBContextResolver.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/JAXBContextResolver.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/JAXBContextResolver.java
deleted file mode 100644
index 5fd0124..0000000
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/JAXBContextResolver.java
+++ /dev/null
@@ -1,64 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.server.applicationhistoryservice.webapp;
-
-import java.util.Arrays;
-import java.util.HashSet;
-import java.util.Set;
-
-import javax.ws.rs.ext.ContextResolver;
-import javax.ws.rs.ext.Provider;
-import javax.xml.bind.JAXBContext;
-
-import org.apache.hadoop.yarn.server.webapp.dao.AppAttemptInfo;
-import org.apache.hadoop.yarn.server.webapp.dao.AppAttemptsInfo;
-import org.apache.hadoop.yarn.server.webapp.dao.AppInfo;
-import org.apache.hadoop.yarn.server.webapp.dao.AppsInfo;
-import org.apache.hadoop.yarn.server.webapp.dao.ContainerInfo;
-import org.apache.hadoop.yarn.server.webapp.dao.ContainersInfo;
-
-import com.google.inject.Singleton;
-import com.sun.jersey.api.json.JSONConfiguration;
-import com.sun.jersey.api.json.JSONJAXBContext;
-
-@Singleton
-@Provider
-@SuppressWarnings("rawtypes")
-public class JAXBContextResolver implements ContextResolver<JAXBContext> {
-
- private JAXBContext context;
- private final Set<Class> types;
-
- // you have to specify all the dao classes here
- private final Class[] cTypes = { AppInfo.class, AppsInfo.class,
- AppAttemptInfo.class, AppAttemptsInfo.class, ContainerInfo.class,
- ContainersInfo.class };
-
- public JAXBContextResolver() throws Exception {
- this.types = new HashSet<Class>(Arrays.asList(cTypes));
- this.context =
- new JSONJAXBContext(JSONConfiguration.natural().rootUnwrapping(false)
- .build(), cTypes);
- }
-
- @Override
- public JAXBContext getContext(Class<?> objectType) {
- return (types.contains(objectType)) ? context : null;
- }
-}
[23/28] ambari git commit: AMBARI-22740 : Fix integration test for
HBase in branch-3.0-ams due to UUID changes. (avijayan)
Posted by av...@apache.org.
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/aggregators/TimelineMetricClusterAggregatorSecondWithCacheSource.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/aggregators/TimelineMetricClusterAggregatorSecondWithCacheSource.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/aggregators/TimelineMetricClusterAggregatorSecondWithCacheSource.java
new file mode 100644
index 0000000..69122f9
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/aggregators/TimelineMetricClusterAggregatorSecondWithCacheSource.java
@@ -0,0 +1,104 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.metrics.core.timeline.aggregators;
+
+import static org.apache.ambari.metrics.core.timeline.aggregators.AggregatorUtils.getTimeSlices;
+
+import java.util.Date;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.ambari.metrics.core.timeline.TimelineMetricDistributedCache;
+import org.apache.ambari.metrics.core.timeline.availability.MetricCollectorHAController;
+import org.apache.ambari.metrics.core.timeline.discovery.TimelineMetricMetadataManager;
+import org.apache.commons.lang.mutable.MutableInt;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.metrics2.sink.timeline.MetricClusterAggregate;
+import org.apache.ambari.metrics.core.timeline.PhoenixHBaseAccessor;
+import org.apache.ambari.metrics.core.timeline.availability.AggregationTaskRunner;
+
+public class TimelineMetricClusterAggregatorSecondWithCacheSource extends TimelineMetricClusterAggregatorSecond {
+ private TimelineMetricDistributedCache distributedCache;
+ public TimelineMetricClusterAggregatorSecondWithCacheSource(AggregationTaskRunner.AGGREGATOR_NAME metricAggregateSecond, TimelineMetricMetadataManager metricMetadataManager, PhoenixHBaseAccessor hBaseAccessor, Configuration metricsConf, String checkpointLocation, long sleepIntervalMillis, int checkpointCutOffMultiplier, String aggregatorDisabledParam, String inputTableName, String outputTableName,
+ Long nativeTimeRangeDelay,
+ Long timeSliceInterval,
+ MetricCollectorHAController haController, TimelineMetricDistributedCache distributedCache) {
+ super(metricAggregateSecond, metricMetadataManager, hBaseAccessor, metricsConf, checkpointLocation, sleepIntervalMillis, checkpointCutOffMultiplier, aggregatorDisabledParam, inputTableName, outputTableName, nativeTimeRangeDelay, timeSliceInterval, haController);
+ this.distributedCache = distributedCache;
+ }
+
+ @Override
+ public boolean doWork(long startTime, long endTime) {
+ LOG.info("Start aggregation cycle @ " + new Date() + ", " +
+ "startTime = " + new Date(startTime) + ", endTime = " + new Date(endTime));
+ try {
+ Map<String, Double> caheMetrics;
+ if (LOG.isDebugEnabled()) {
+ caheMetrics = distributedCache.getPointInTimeCacheMetrics();
+ LOG.debug("Ignite metrics before eviction : " + caheMetrics);
+ }
+
+ LOG.info("Trying to evict elements from cache");
+ Map<TimelineClusterMetric, MetricClusterAggregate> metricsFromCache = distributedCache.evictMetricAggregates(startTime - serverTimeShiftAdjustment, endTime - serverTimeShiftAdjustment);
+ LOG.info(String.format("Evicted %s elements from cache.", metricsFromCache.size()));
+
+ if (LOG.isDebugEnabled()) {
+ caheMetrics = distributedCache.getPointInTimeCacheMetrics();
+ LOG.debug("Ignite metrics after eviction : " + caheMetrics);
+ }
+
+ List<Long[]> timeSlices = getTimeSlices(startTime - serverTimeShiftAdjustment, endTime - serverTimeShiftAdjustment, timeSliceIntervalMillis);
+ Map<TimelineClusterMetric, MetricClusterAggregate> result = aggregateMetricsFromMetricClusterAggregates(metricsFromCache, timeSlices);
+
+ LOG.info("Saving " + result.size() + " metric aggregates.");
+ hBaseAccessor.saveClusterAggregateRecords(result);
+ LOG.info("End aggregation cycle @ " + new Date());
+ return true;
+ } catch (Exception e) {
+ LOG.error("Exception during aggregation. ", e);
+ return false;
+ }
+ }
+
+ //Slices in cache could be different from aggregate slices, so need to recalculate. Counts hosted apps
+ Map<TimelineClusterMetric, MetricClusterAggregate> aggregateMetricsFromMetricClusterAggregates(Map<TimelineClusterMetric, MetricClusterAggregate> metricsFromCache, List<Long[]> timeSlices) {
+ //TODO add basic interpolation
+ //TODO investigate if needed, maybe add config to disable/enable
+ //count hosted apps
+ Map<String, MutableInt> hostedAppCounter = new HashMap<>();
+ for (Map.Entry<TimelineClusterMetric, MetricClusterAggregate> clusterMetricAggregateEntry : metricsFromCache.entrySet()) {
+ int numHosts = clusterMetricAggregateEntry.getValue().getNumberOfHosts();
+ String appId = clusterMetricAggregateEntry.getKey().getAppId();
+ if (!hostedAppCounter.containsKey(appId)) {
+ hostedAppCounter.put(appId, new MutableInt(numHosts));
+ } else {
+ int currentHostCount = hostedAppCounter.get(appId).intValue();
+ if (currentHostCount < numHosts) {
+ hostedAppCounter.put(appId, new MutableInt(numHosts));
+ }
+ }
+ }
+
+ // Add liveHosts per AppId metrics.
+ processLiveAppCountMetrics(metricsFromCache, hostedAppCounter, timeSlices.get(timeSlices.size() - 1)[1]);
+
+ return metricsFromCache;
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/aggregators/TimelineMetricFilteringHostAggregator.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/aggregators/TimelineMetricFilteringHostAggregator.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/aggregators/TimelineMetricFilteringHostAggregator.java
new file mode 100644
index 0000000..371d9fa
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/aggregators/TimelineMetricFilteringHostAggregator.java
@@ -0,0 +1,93 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.metrics.core.timeline.aggregators;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+
+import org.apache.ambari.metrics.core.timeline.availability.AggregationTaskRunner;
+import org.apache.ambari.metrics.core.timeline.availability.MetricCollectorHAController;
+import org.apache.ambari.metrics.core.timeline.discovery.TimelineMetricMetadataManager;
+import org.apache.ambari.metrics.core.timeline.query.Condition;
+import org.apache.ambari.metrics.core.timeline.query.DefaultCondition;
+import org.apache.ambari.metrics.core.timeline.query.PhoenixTransactSQL;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.ambari.metrics.core.timeline.PhoenixHBaseAccessor;
+
+public class TimelineMetricFilteringHostAggregator extends TimelineMetricHostAggregator {
+ private static final Log LOG = LogFactory.getLog(TimelineMetricFilteringHostAggregator.class);
+ private TimelineMetricMetadataManager metricMetadataManager;
+ private ConcurrentHashMap<String, Long> postedAggregatedMap;
+
+ public TimelineMetricFilteringHostAggregator(AggregationTaskRunner.AGGREGATOR_NAME aggregatorName,
+ TimelineMetricMetadataManager metricMetadataManager,
+ PhoenixHBaseAccessor hBaseAccessor,
+ Configuration metricsConf,
+ String checkpointLocation,
+ Long sleepIntervalMillis,
+ Integer checkpointCutOffMultiplier,
+ String hostAggregatorDisabledParam,
+ String tableName,
+ String outputTableName,
+ Long nativeTimeRangeDelay,
+ MetricCollectorHAController haController,
+ ConcurrentHashMap<String, Long> postedAggregatedMap) {
+ super(aggregatorName, metricMetadataManager,
+ hBaseAccessor, metricsConf,
+ checkpointLocation,
+ sleepIntervalMillis,
+ checkpointCutOffMultiplier,
+ hostAggregatorDisabledParam,
+ tableName,
+ outputTableName,
+ nativeTimeRangeDelay,
+ haController);
+ this.metricMetadataManager = metricMetadataManager;
+ this.postedAggregatedMap = postedAggregatedMap;
+ }
+
+ @Override
+ protected Condition prepareMetricQueryCondition(long startTime, long endTime) {
+ List<String> aggregatedHostnames = new ArrayList<>();
+ for (Map.Entry<String, Long> entry : postedAggregatedMap.entrySet()) {
+ if (entry.getValue() > startTime && entry.getValue() <= endTime) {
+ aggregatedHostnames.add(entry.getKey());
+ }
+ }
+ List<String> notAggregatedHostnames = metricMetadataManager.getNotLikeHostnames(aggregatedHostnames);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Already aggregated hostnames based on postedAggregatedMap : " + aggregatedHostnames);
+ LOG.debug("Hostnames that will be aggregated : " + notAggregatedHostnames);
+ }
+ List<byte[]> uuids = metricMetadataManager.getUuids(new ArrayList<String>(), notAggregatedHostnames, "", "");
+
+ Condition condition = new DefaultCondition(uuids, null, null, null, null, startTime,
+ endTime, null, null, true);
+ condition.setNoLimit();
+ condition.setFetchSize(resultsetFetchSize);
+ condition.setStatement(String.format(PhoenixTransactSQL.GET_METRIC_AGGREGATE_ONLY_SQL, tableName));
+ // Retaining order of the row-key avoids client side merge sort.
+ condition.addOrderByColumn("UUID");
+ condition.addOrderByColumn("SERVER_TIME");
+ return condition;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/aggregators/TimelineMetricHostAggregator.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/aggregators/TimelineMetricHostAggregator.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/aggregators/TimelineMetricHostAggregator.java
new file mode 100644
index 0000000..c25d6ce
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/aggregators/TimelineMetricHostAggregator.java
@@ -0,0 +1,120 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.metrics.core.timeline.aggregators;
+
+import java.io.IOException;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.ambari.metrics.core.timeline.availability.MetricCollectorHAController;
+import org.apache.ambari.metrics.core.timeline.discovery.TimelineMetricMetadataManager;
+import org.apache.ambari.metrics.core.timeline.query.Condition;
+import org.apache.ambari.metrics.core.timeline.query.DefaultCondition;
+import org.apache.ambari.metrics.core.timeline.query.PhoenixTransactSQL;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.metrics2.sink.timeline.MetricHostAggregate;
+import org.apache.hadoop.metrics2.sink.timeline.TimelineMetric;
+import org.apache.ambari.metrics.core.timeline.PhoenixHBaseAccessor;
+import org.apache.ambari.metrics.core.timeline.availability.AggregationTaskRunner.AGGREGATOR_NAME;
+
+public class TimelineMetricHostAggregator extends AbstractTimelineAggregator {
+ private static final Log LOG = LogFactory.getLog(TimelineMetricHostAggregator.class);
+ TimelineMetricReadHelper readHelper;
+
+ public TimelineMetricHostAggregator(AGGREGATOR_NAME aggregatorName,
+ TimelineMetricMetadataManager metricMetadataManager,
+ PhoenixHBaseAccessor hBaseAccessor,
+ Configuration metricsConf,
+ String checkpointLocation,
+ Long sleepIntervalMillis,
+ Integer checkpointCutOffMultiplier,
+ String hostAggregatorDisabledParam,
+ String tableName,
+ String outputTableName,
+ Long nativeTimeRangeDelay,
+ MetricCollectorHAController haController) {
+ super(aggregatorName, hBaseAccessor, metricsConf, checkpointLocation,
+ sleepIntervalMillis, checkpointCutOffMultiplier, hostAggregatorDisabledParam,
+ tableName, outputTableName, nativeTimeRangeDelay, haController);
+ readHelper = new TimelineMetricReadHelper(metricMetadataManager, false);
+ }
+
+ @Override
+ protected void aggregate(ResultSet rs, long startTime, long endTime) throws IOException, SQLException {
+
+ Map<TimelineMetric, MetricHostAggregate> hostAggregateMap = aggregateMetricsFromResultSet(rs, endTime);
+
+ LOG.info("Saving " + hostAggregateMap.size() + " metric aggregates.");
+ hBaseAccessor.saveHostAggregateRecords(hostAggregateMap, outputTableName);
+ }
+
+ @Override
+ protected Condition prepareMetricQueryCondition(long startTime, long endTime) {
+ Condition condition = new DefaultCondition(null, null, null, null, startTime,
+ endTime, null, null, true);
+ condition.setNoLimit();
+ condition.setFetchSize(resultsetFetchSize);
+ condition.setStatement(String.format(PhoenixTransactSQL.GET_METRIC_AGGREGATE_ONLY_SQL, tableName));
+ // Retaining order of the row-key avoids client side merge sort.
+ condition.addOrderByColumn("UUID");
+ condition.addOrderByColumn("SERVER_TIME");
+ return condition;
+ }
+
+ private Map<TimelineMetric, MetricHostAggregate> aggregateMetricsFromResultSet(ResultSet rs, long endTime)
+ throws IOException, SQLException {
+ TimelineMetric existingMetric = null;
+ MetricHostAggregate hostAggregate = null;
+ Map<TimelineMetric, MetricHostAggregate> hostAggregateMap = new HashMap<TimelineMetric, MetricHostAggregate>();
+
+
+ while (rs.next()) {
+ TimelineMetric currentMetric =
+ readHelper.getTimelineMetricKeyFromResultSet(rs);
+ MetricHostAggregate currentHostAggregate =
+ readHelper.getMetricHostAggregateFromResultSet(rs);
+
+ if (existingMetric == null) {
+ // First row
+ existingMetric = currentMetric;
+ currentMetric.setStartTime(endTime);
+ hostAggregate = new MetricHostAggregate();
+ hostAggregateMap.put(currentMetric, hostAggregate);
+ }
+
+ if (existingMetric.equalsExceptTime(currentMetric)) {
+ // Recalculate totals with current metric
+ hostAggregate.updateAggregates(currentHostAggregate);
+ } else {
+ // Switched over to a new metric - save existing - create new aggregate
+ currentMetric.setStartTime(endTime);
+ hostAggregate = new MetricHostAggregate();
+ hostAggregate.updateAggregates(currentHostAggregate);
+ hostAggregateMap.put(currentMetric, hostAggregate);
+ existingMetric = currentMetric;
+ }
+ }
+ return hostAggregateMap;
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/aggregators/TimelineMetricReadHelper.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/aggregators/TimelineMetricReadHelper.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/aggregators/TimelineMetricReadHelper.java
new file mode 100644
index 0000000..a56c7aa
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/aggregators/TimelineMetricReadHelper.java
@@ -0,0 +1,170 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.metrics.core.timeline.aggregators;
+
+
+import java.io.IOException;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.util.TreeMap;
+
+import org.apache.hadoop.metrics2.sink.timeline.MetricClusterAggregate;
+import org.apache.hadoop.metrics2.sink.timeline.MetricHostAggregate;
+import org.apache.hadoop.metrics2.sink.timeline.SingleValuedTimelineMetric;
+import org.apache.hadoop.metrics2.sink.timeline.TimelineMetric;
+import org.apache.ambari.metrics.core.timeline.PhoenixHBaseAccessor;
+import org.apache.ambari.metrics.core.timeline.discovery.TimelineMetricMetadataManager;
+
+public class TimelineMetricReadHelper {
+
+ private boolean ignoreInstance = false;
+ private TimelineMetricMetadataManager metadataManagerInstance = null;
+
+ public TimelineMetricReadHelper() {}
+
+ public TimelineMetricReadHelper(boolean ignoreInstance) {
+ this.ignoreInstance = ignoreInstance;
+ }
+
+ public TimelineMetricReadHelper(TimelineMetricMetadataManager timelineMetricMetadataManager) {
+ this.metadataManagerInstance = timelineMetricMetadataManager;
+ }
+
+ public TimelineMetricReadHelper(TimelineMetricMetadataManager timelineMetricMetadataManager, boolean ignoreInstance) {
+ this.metadataManagerInstance = timelineMetricMetadataManager;
+ this.ignoreInstance = ignoreInstance;
+ }
+
+ public TimelineMetric getTimelineMetricFromResultSet(ResultSet rs)
+ throws SQLException, IOException {
+ TimelineMetric metric = getTimelineMetricCommonsFromResultSet(rs);
+ TreeMap<Long, Double> sortedByTimeMetrics = PhoenixHBaseAccessor.readMetricFromJSON(rs.getString("METRICS"));
+ metric.setMetricValues(sortedByTimeMetrics);
+ return metric;
+ }
+
+ public SingleValuedTimelineMetric getAggregatedTimelineMetricFromResultSet(ResultSet rs,
+ Function f) throws SQLException, IOException {
+
+ byte[] uuid = rs.getBytes("UUID");
+ TimelineMetric timelineMetric = metadataManagerInstance.getMetricFromUuid(uuid);
+ Function function = (f != null) ? f : Function.DEFAULT_VALUE_FUNCTION;
+ SingleValuedTimelineMetric metric = new SingleValuedTimelineMetric(
+ timelineMetric.getMetricName() + function.getSuffix(),
+ timelineMetric.getAppId(),
+ timelineMetric.getInstanceId(),
+ timelineMetric.getHostName(),
+ rs.getLong("SERVER_TIME")
+ );
+
+ double value;
+ switch(function.getReadFunction()){
+ case AVG:
+ value = rs.getDouble("METRIC_SUM") / rs.getInt("METRIC_COUNT");
+ break;
+ case MIN:
+ value = rs.getDouble("METRIC_MIN");
+ break;
+ case MAX:
+ value = rs.getDouble("METRIC_MAX");
+ break;
+ case SUM:
+ value = rs.getDouble("METRIC_SUM") / rs.getInt("METRIC_COUNT");
+ break;
+ default:
+ value = rs.getDouble("METRIC_SUM") / rs.getInt("METRIC_COUNT");
+ break;
+ }
+
+ metric.setSingleTimeseriesValue(rs.getLong("SERVER_TIME"), value);
+
+ return metric;
+ }
+
+ /**
+ * Returns common part of timeline metrics record without the values.
+ */
+ public TimelineMetric getTimelineMetricCommonsFromResultSet(ResultSet rs)
+ throws SQLException {
+
+ byte[] uuid = rs.getBytes("UUID");
+ TimelineMetric metric = metadataManagerInstance.getMetricFromUuid(uuid);
+ if (ignoreInstance) {
+ metric.setInstanceId(null);
+ }
+ metric.setStartTime(rs.getLong("SERVER_TIME"));
+ return metric;
+ }
+
+ public MetricClusterAggregate getMetricClusterAggregateFromResultSet(ResultSet rs)
+ throws SQLException {
+ MetricClusterAggregate agg = new MetricClusterAggregate();
+ agg.setSum(rs.getDouble("METRIC_SUM"));
+ agg.setMax(rs.getDouble("METRIC_MAX"));
+ agg.setMin(rs.getDouble("METRIC_MIN"));
+ agg.setNumberOfHosts(rs.getInt("HOSTS_COUNT"));
+
+ agg.setDeviation(0.0);
+
+ return agg;
+ }
+
+ public MetricClusterAggregate getMetricClusterTimeAggregateFromResultSet(ResultSet rs)
+ throws SQLException {
+ MetricClusterAggregate agg = new MetricClusterAggregate();
+ agg.setSum(rs.getDouble("METRIC_SUM"));
+ agg.setMax(rs.getDouble("METRIC_MAX"));
+ agg.setMin(rs.getDouble("METRIC_MIN"));
+ agg.setNumberOfHosts(rs.getInt("METRIC_COUNT"));
+
+ agg.setDeviation(0.0);
+
+ return agg;
+ }
+
+ public TimelineClusterMetric fromResultSet(ResultSet rs) throws SQLException {
+
+ byte[] uuid = rs.getBytes("UUID");
+ TimelineMetric timelineMetric = metadataManagerInstance.getMetricFromUuid(uuid);
+
+ return new TimelineClusterMetric(
+ timelineMetric.getMetricName(),
+ timelineMetric.getAppId(),
+ ignoreInstance ? null : timelineMetric.getInstanceId(),
+ rs.getLong("SERVER_TIME"));
+ }
+
+ public MetricHostAggregate getMetricHostAggregateFromResultSet(ResultSet rs)
+ throws SQLException {
+ MetricHostAggregate metricHostAggregate = new MetricHostAggregate();
+ metricHostAggregate.setSum(rs.getDouble("METRIC_SUM"));
+ metricHostAggregate.setMax(rs.getDouble("METRIC_MAX"));
+ metricHostAggregate.setMin(rs.getDouble("METRIC_MIN"));
+ metricHostAggregate.setNumberOfSamples(rs.getLong("METRIC_COUNT"));
+
+ metricHostAggregate.setDeviation(0.0);
+ return metricHostAggregate;
+ }
+
+ public TimelineMetric getTimelineMetricKeyFromResultSet(ResultSet rs)
+ throws SQLException, IOException {
+ byte[] uuid = rs.getBytes("UUID");
+ return metadataManagerInstance.getMetricFromUuid(uuid);
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/aggregators/TopNDownSampler.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/aggregators/TopNDownSampler.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/aggregators/TopNDownSampler.java
new file mode 100644
index 0000000..e28d465
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/aggregators/TopNDownSampler.java
@@ -0,0 +1,112 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.metrics.core.timeline.aggregators;
+
+import static org.apache.ambari.metrics.core.timeline.query.PhoenixTransactSQL.TOPN_DOWNSAMPLER_CLUSTER_METRIC_SELECT_SQL;
+import static org.apache.ambari.metrics.core.timeline.query.PhoenixTransactSQL.TOPN_DOWNSAMPLER_HOST_METRIC_SELECT_SQL;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.ambari.metrics.core.timeline.query.TopNCondition;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.metrics2.sink.timeline.TopNConfig;
+
+public class TopNDownSampler implements CustomDownSampler {
+
+ private TopNConfig topNConfig;
+ private static final Log LOG = LogFactory.getLog(TopNDownSampler.class);
+ protected String metricPatterns;
+
+ public static TopNDownSampler fromConfig(Map<String, String> conf) {
+ String metricPatterns = conf.get(DownSamplerUtils.downSamplerConfigPrefix + "topn." +
+ DownSamplerUtils.downSamplerMetricPatternsConfig);
+
+ String topNString = conf.get(DownSamplerUtils.downSamplerConfigPrefix + "topn.value");
+ Integer topNValue = topNString != null ? Integer.valueOf(topNString) : 10;
+ String topNFunction = conf.get(DownSamplerUtils.downSamplerConfigPrefix + "topn.function");
+
+ return new TopNDownSampler(new TopNConfig(topNValue, topNFunction, false), metricPatterns);
+ }
+
+ public TopNDownSampler(TopNConfig topNConfig, String metricPatterns) {
+ this.topNConfig = topNConfig;
+ this.metricPatterns = metricPatterns;
+ }
+
+ @Override
+ public boolean validateConfigs() {
+ if (topNConfig == null) {
+ return false;
+ }
+
+ if (topNConfig.getTopN() <= 0) {
+ return false;
+ }
+
+ if (StringUtils.isEmpty(metricPatterns)) {
+ return false;
+ }
+
+ return true;
+ }
+
+ /**
+ * Prepare downsampling SELECT statement(s) used to determine the data to be written into the Aggregate table.
+ * @param startTime
+ * @param endTime
+ * @param tableName
+ * @return
+ */
+ @Override
+ public List<String> prepareDownSamplingStatement(Long startTime, Long endTime, String tableName) {
+ List<String> stmts = new ArrayList<>();
+
+ Function.ReadFunction readFunction = Function.ReadFunction.getFunction(topNConfig.getTopNFunction());
+ Function function = new Function(readFunction, null);
+ String columnSelect = TopNCondition.getColumnSelect(function);
+
+ List<String> metricPatternList = Arrays.asList(metricPatterns.split(","));
+
+ for (String metricPattern : metricPatternList) {
+ String metricPatternClause = "'" + metricPattern + "'";
+ //TODO : Need a better way to find out what kind of aggregation the current one is.
+ if (tableName.contains("RECORD")) {
+ stmts.add(String.format(TOPN_DOWNSAMPLER_HOST_METRIC_SELECT_SQL,
+ endTime, columnSelect, columnSelect, columnSelect, tableName, metricPatternClause,
+ startTime, endTime, columnSelect, topNConfig.getTopN()));
+ } else {
+ stmts.add(String.format(TOPN_DOWNSAMPLER_CLUSTER_METRIC_SELECT_SQL,
+ endTime, columnSelect, columnSelect, columnSelect, tableName, metricPatternClause,
+ startTime, endTime, columnSelect, topNConfig.getTopN()));
+ }
+ }
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("DownSampling Stmt: " + stmts.toString());
+ }
+
+ return stmts;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/aggregators/v2/TimelineMetricClusterAggregator.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/aggregators/v2/TimelineMetricClusterAggregator.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/aggregators/v2/TimelineMetricClusterAggregator.java
new file mode 100644
index 0000000..9c255e7
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/aggregators/v2/TimelineMetricClusterAggregator.java
@@ -0,0 +1,93 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.metrics.core.timeline.aggregators.v2;
+
+import static org.apache.ambari.metrics.core.timeline.query.PhoenixTransactSQL.GET_AGGREGATED_APP_METRIC_GROUPBY_SQL;
+import static org.apache.ambari.metrics.core.timeline.query.PhoenixTransactSQL.METRICS_CLUSTER_AGGREGATE_TABLE_NAME;
+
+import java.io.IOException;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.util.Date;
+
+import org.apache.ambari.metrics.core.timeline.availability.MetricCollectorHAController;
+import org.apache.ambari.metrics.core.timeline.query.EmptyCondition;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.ambari.metrics.core.timeline.PhoenixHBaseAccessor;
+import org.apache.ambari.metrics.core.timeline.aggregators.AbstractTimelineAggregator;
+import org.apache.ambari.metrics.core.timeline.availability.AggregationTaskRunner.AGGREGATOR_NAME;
+import org.apache.ambari.metrics.core.timeline.query.Condition;
+
+public class TimelineMetricClusterAggregator extends AbstractTimelineAggregator {
+ private final String aggregateColumnName;
+
+ public TimelineMetricClusterAggregator(AGGREGATOR_NAME aggregatorName,
+ PhoenixHBaseAccessor hBaseAccessor,
+ Configuration metricsConf,
+ String checkpointLocation,
+ Long sleepIntervalMillis,
+ Integer checkpointCutOffMultiplier,
+ String hostAggregatorDisabledParam,
+ String inputTableName,
+ String outputTableName,
+ Long nativeTimeRangeDelay,
+ MetricCollectorHAController haController) {
+ super(aggregatorName, hBaseAccessor, metricsConf, checkpointLocation,
+ sleepIntervalMillis, checkpointCutOffMultiplier,
+ hostAggregatorDisabledParam, inputTableName, outputTableName,
+ nativeTimeRangeDelay, haController);
+
+ if (inputTableName.equals(METRICS_CLUSTER_AGGREGATE_TABLE_NAME)) {
+ aggregateColumnName = "HOSTS_COUNT";
+ } else {
+ aggregateColumnName = "METRIC_COUNT";
+ }
+ }
+
+ @Override
+ protected Condition prepareMetricQueryCondition(long startTime, long endTime) {
+ EmptyCondition condition = new EmptyCondition();
+ condition.setDoUpdate(true);
+
+ /*
+ UPSERT INTO METRIC_AGGREGATE_HOURLY (METRIC_NAME, APP_ID, INSTANCE_ID,
+ SERVER_TIME, UNITS, METRIC_SUM, METRIC_COUNT, METRIC_MAX, METRIC_MIN)
+ SELECT METRIC_NAME, APP_ID, INSTANCE_ID, MAX(SERVER_TIME), UNITS,
+ SUM(METRIC_SUM), SUM(HOSTS_COUNT), MAX(METRIC_MAX), MIN(METRIC_MIN)
+ FROM METRIC_AGGREGATE WHERE SERVER_TIME >= 1441155600000 AND
+ SERVER_TIME < 1441159200000 GROUP BY METRIC_NAME, APP_ID, INSTANCE_ID, UNITS;
+ */
+
+ condition.setStatement(String.format(GET_AGGREGATED_APP_METRIC_GROUPBY_SQL,
+ outputTableName, endTime, aggregateColumnName, tableName,
+ getDownsampledMetricSkipClause(), startTime, endTime));
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Condition: " + condition.toString());
+ }
+
+ return condition;
+ }
+
+ @Override
+ protected void aggregate(ResultSet rs, long startTime, long endTime) throws IOException, SQLException {
+ LOG.info("Aggregated cluster metrics for " + outputTableName +
+ ", with startTime = " + new Date(startTime) +
+ ", endTime = " + new Date(endTime));
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/aggregators/v2/TimelineMetricFilteringHostAggregator.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/aggregators/v2/TimelineMetricFilteringHostAggregator.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/aggregators/v2/TimelineMetricFilteringHostAggregator.java
new file mode 100644
index 0000000..1026cbe
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/aggregators/v2/TimelineMetricFilteringHostAggregator.java
@@ -0,0 +1,119 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.metrics.core.timeline.aggregators.v2;
+
+import static org.apache.ambari.metrics.core.timeline.query.PhoenixTransactSQL.GET_AGGREGATED_HOST_METRIC_GROUPBY_SQL;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+
+import org.apache.ambari.metrics.core.timeline.availability.MetricCollectorHAController;
+import org.apache.ambari.metrics.core.timeline.query.EmptyCondition;
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.ambari.metrics.core.timeline.PhoenixHBaseAccessor;
+import org.apache.ambari.metrics.core.timeline.availability.AggregationTaskRunner;
+import org.apache.ambari.metrics.core.timeline.discovery.TimelineMetricMetadataManager;
+import org.apache.ambari.metrics.core.timeline.query.Condition;
+
+public class TimelineMetricFilteringHostAggregator extends TimelineMetricHostAggregator {
+ private TimelineMetricMetadataManager metricMetadataManager;
+ private ConcurrentHashMap<String, Long> postedAggregatedMap;
+
+ public TimelineMetricFilteringHostAggregator(AggregationTaskRunner.AGGREGATOR_NAME aggregatorName,
+ TimelineMetricMetadataManager metricMetadataManager,
+ PhoenixHBaseAccessor hBaseAccessor,
+ Configuration metricsConf,
+ String checkpointLocation,
+ Long sleepIntervalMillis,
+ Integer checkpointCutOffMultiplier,
+ String hostAggregatorDisabledParam,
+ String tableName,
+ String outputTableName,
+ Long nativeTimeRangeDelay,
+ MetricCollectorHAController haController,
+ ConcurrentHashMap<String, Long> postedAggregatedMap) {
+ super(aggregatorName,
+ hBaseAccessor, metricsConf,
+ checkpointLocation,
+ sleepIntervalMillis,
+ checkpointCutOffMultiplier,
+ hostAggregatorDisabledParam,
+ tableName,
+ outputTableName,
+ nativeTimeRangeDelay,
+ haController);
+ this.metricMetadataManager = metricMetadataManager;
+ this.postedAggregatedMap = postedAggregatedMap;
+ }
+
+ @Override
+ protected Condition prepareMetricQueryCondition(long startTime, long endTime) {
+ List<String> aggregatedHostnames = new ArrayList<>();
+ for (Map.Entry<String, Long> entry : postedAggregatedMap.entrySet()) {
+ if (entry.getValue() > startTime && entry.getValue() <= endTime) {
+ aggregatedHostnames.add(entry.getKey());
+ }
+ }
+ List<String> notAggregatedHostnames = metricMetadataManager.getNotLikeHostnames(aggregatedHostnames);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Already aggregated hostnames based on postedAggregatedMap : " + aggregatedHostnames);
+ LOG.debug("Hostnames that will be aggregated : " + notAggregatedHostnames);
+ }
+ List<byte[]> uuids = metricMetadataManager.getUuids(new ArrayList<String>(), notAggregatedHostnames, "", "");
+
+ EmptyCondition condition = new EmptyCondition();
+ condition.setDoUpdate(true);
+
+ condition.setStatement(String.format(GET_AGGREGATED_HOST_METRIC_GROUPBY_SQL,
+ outputTableName, endTime, tableName,
+ getDownsampledMetricSkipClause() + getIncludedUuidsClause(uuids), startTime, endTime));
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Condition: " + condition.toString());
+ }
+
+ return condition;
+ }
+
+ private String getIncludedUuidsClause(List<byte[]> uuids) {
+ StringBuilder sb = new StringBuilder();
+ sb.append("(");
+
+ //LIKE clause
+ // (UUID LIKE ? OR UUID LIKE ?) AND
+ if (CollectionUtils.isNotEmpty(uuids)) {
+ for (int i = 0; i < uuids.size(); i++) {
+ sb.append("UUID");
+ sb.append(" LIKE ");
+ sb.append("'%");
+ sb.append(new String(uuids.get(i)));
+ sb.append("'");
+
+ if (i == uuids.size() - 1) {
+ sb.append(") AND ");
+ } else {
+ sb.append(" OR ");
+ }
+ }
+ }
+ return sb.toString();
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/aggregators/v2/TimelineMetricHostAggregator.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/aggregators/v2/TimelineMetricHostAggregator.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/aggregators/v2/TimelineMetricHostAggregator.java
new file mode 100644
index 0000000..9e8df6d
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/aggregators/v2/TimelineMetricHostAggregator.java
@@ -0,0 +1,75 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.metrics.core.timeline.aggregators.v2;
+
+import java.io.IOException;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.util.Date;
+
+import org.apache.ambari.metrics.core.timeline.aggregators.AbstractTimelineAggregator;
+import org.apache.ambari.metrics.core.timeline.availability.AggregationTaskRunner;
+import org.apache.ambari.metrics.core.timeline.availability.MetricCollectorHAController;
+import org.apache.ambari.metrics.core.timeline.query.Condition;
+import org.apache.ambari.metrics.core.timeline.query.EmptyCondition;
+import org.apache.ambari.metrics.core.timeline.query.PhoenixTransactSQL;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.ambari.metrics.core.timeline.PhoenixHBaseAccessor;
+
+public class TimelineMetricHostAggregator extends AbstractTimelineAggregator {
+
+ public TimelineMetricHostAggregator(AggregationTaskRunner.AGGREGATOR_NAME aggregatorName,
+ PhoenixHBaseAccessor hBaseAccessor,
+ Configuration metricsConf,
+ String checkpointLocation,
+ Long sleepIntervalMillis,
+ Integer checkpointCutOffMultiplier,
+ String hostAggregatorDisabledParam,
+ String tableName,
+ String outputTableName,
+ Long nativeTimeRangeDelay,
+ MetricCollectorHAController haController) {
+ super(aggregatorName, hBaseAccessor, metricsConf, checkpointLocation,
+ sleepIntervalMillis, checkpointCutOffMultiplier, hostAggregatorDisabledParam,
+ tableName, outputTableName, nativeTimeRangeDelay, haController);
+ }
+
+ @Override
+ protected void aggregate(ResultSet rs, long startTime, long endTime) throws IOException, SQLException {
+
+ LOG.info("Aggregated host metrics for " + outputTableName +
+ ", with startTime = " + new Date(startTime) +
+ ", endTime = " + new Date(endTime));
+ }
+
+ @Override
+ protected Condition prepareMetricQueryCondition(long startTime, long endTime) {
+ EmptyCondition condition = new EmptyCondition();
+ condition.setDoUpdate(true);
+
+ condition.setStatement(String.format(PhoenixTransactSQL.GET_AGGREGATED_HOST_METRIC_GROUPBY_SQL,
+ outputTableName, endTime, tableName,
+ getDownsampledMetricSkipClause(), startTime, endTime));
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Condition: " + condition.toString());
+ }
+
+ return condition;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/availability/AggregationTaskRunner.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/availability/AggregationTaskRunner.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/availability/AggregationTaskRunner.java
new file mode 100644
index 0000000..9a27d55
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/availability/AggregationTaskRunner.java
@@ -0,0 +1,141 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.metrics.core.timeline.availability;
+
+import static org.apache.ambari.metrics.core.timeline.availability.AggregationTaskRunner.AGGREGATOR_NAME.METRIC_AGGREGATE_DAILY;
+import static org.apache.ambari.metrics.core.timeline.availability.AggregationTaskRunner.AGGREGATOR_NAME.METRIC_AGGREGATE_HOURLY;
+import static org.apache.ambari.metrics.core.timeline.availability.AggregationTaskRunner.AGGREGATOR_NAME.METRIC_AGGREGATE_MINUTE;
+import static org.apache.ambari.metrics.core.timeline.availability.AggregationTaskRunner.AGGREGATOR_NAME.METRIC_AGGREGATE_SECOND;
+import static org.apache.ambari.metrics.core.timeline.availability.AggregationTaskRunner.AGGREGATOR_NAME.METRIC_RECORD_DAILY;
+import static org.apache.ambari.metrics.core.timeline.availability.AggregationTaskRunner.AGGREGATOR_NAME.METRIC_RECORD_HOURLY;
+import static org.apache.ambari.metrics.core.timeline.availability.AggregationTaskRunner.AGGREGATOR_NAME.METRIC_RECORD_MINUTE;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import org.apache.ambari.metrics.core.timeline.aggregators.TimelineMetricAggregator;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.helix.HelixManager;
+import org.apache.helix.HelixManagerFactory;
+import org.apache.helix.InstanceType;
+import org.apache.helix.participant.StateMachineEngine;
+
+public class AggregationTaskRunner {
+ private final String instanceName;
+ private final String zkAddress;
+ private final String clusterName;
+ private HelixManager manager;
+ private static final Log LOG = LogFactory.getLog(AggregationTaskRunner.class);
+ private CheckpointManager checkpointManager;
+ // Map partition name to an aggregator dimension
+ static final Map<String, TimelineMetricAggregator.AGGREGATOR_TYPE> PARTITION_AGGREGATION_TYPES = new HashMap<>();
+ // Ownership flags to be set by the State transitions
+ private final AtomicBoolean performsClusterAggregation = new AtomicBoolean(false);
+ private final AtomicBoolean performsHostAggregation = new AtomicBoolean(false);
+
+ public enum AGGREGATOR_NAME {
+ METRIC_RECORD_MINUTE,
+ METRIC_RECORD_HOURLY,
+ METRIC_RECORD_DAILY,
+ METRIC_AGGREGATE_SECOND,
+ METRIC_AGGREGATE_MINUTE,
+ METRIC_AGGREGATE_HOURLY,
+ METRIC_AGGREGATE_DAILY,
+ }
+
+ public static final Map<AGGREGATOR_NAME, String> ACTUAL_AGGREGATOR_NAMES = new HashMap<>();
+
+ static {
+ ACTUAL_AGGREGATOR_NAMES.put(METRIC_RECORD_MINUTE, "TimelineMetricHostAggregatorMinute");
+ ACTUAL_AGGREGATOR_NAMES.put(METRIC_RECORD_HOURLY, "TimelineMetricHostAggregatorHourly");
+ ACTUAL_AGGREGATOR_NAMES.put(METRIC_RECORD_DAILY, "TimelineMetricHostAggregatorDaily");
+ ACTUAL_AGGREGATOR_NAMES.put(METRIC_AGGREGATE_SECOND, "TimelineClusterAggregatorSecond");
+ ACTUAL_AGGREGATOR_NAMES.put(METRIC_AGGREGATE_MINUTE, "TimelineClusterAggregatorMinute");
+ ACTUAL_AGGREGATOR_NAMES.put(METRIC_AGGREGATE_HOURLY, "TimelineClusterAggregatorHourly");
+ ACTUAL_AGGREGATOR_NAMES.put(METRIC_AGGREGATE_DAILY, "TimelineClusterAggregatorDaily");
+
+ // Partition name to task assignment
+ PARTITION_AGGREGATION_TYPES.put(MetricCollectorHAController.METRIC_AGGREGATORS + "_0", TimelineMetricAggregator.AGGREGATOR_TYPE.CLUSTER);
+ PARTITION_AGGREGATION_TYPES.put(MetricCollectorHAController.METRIC_AGGREGATORS + "_1", TimelineMetricAggregator.AGGREGATOR_TYPE.HOST);
+ }
+
+ public AggregationTaskRunner(String instanceName, String zkAddress, String clusterName) {
+ this.instanceName = instanceName;
+ this.zkAddress = zkAddress;
+ this.clusterName = clusterName;
+ }
+
+ public void initialize() throws Exception {
+ manager = HelixManagerFactory.getZKHelixManager(clusterName, instanceName,
+ InstanceType.PARTICIPANT, zkAddress);
+
+ OnlineOfflineStateModelFactory stateModelFactory =
+ new OnlineOfflineStateModelFactory(instanceName, this);
+
+ StateMachineEngine stateMach = manager.getStateMachineEngine();
+ stateMach.registerStateModelFactory(MetricCollectorHAController.DEFAULT_STATE_MODEL, stateModelFactory);
+ manager.connect();
+
+ checkpointManager = new CheckpointManager(manager.getHelixPropertyStore());
+ }
+
+ public boolean performsClusterAggregation() {
+ return performsClusterAggregation.get();
+ }
+
+ public boolean performsHostAggregation() {
+ return performsHostAggregation.get();
+ }
+
+ public CheckpointManager getCheckpointManager() {
+ return checkpointManager;
+ }
+
+ public void setPartitionAggregationFunction(TimelineMetricAggregator.AGGREGATOR_TYPE type) {
+ switch (type) {
+ case HOST:
+ performsHostAggregation.set(true);
+ LOG.info("Set host aggregator function for : " + instanceName);
+ break;
+ case CLUSTER:
+ performsClusterAggregation.set(true);
+ LOG.info("Set cluster aggregator function for : " + instanceName);
+ }
+ }
+
+ public void unsetPartitionAggregationFunction(TimelineMetricAggregator.AGGREGATOR_TYPE type) {
+ switch (type) {
+ case HOST:
+ performsHostAggregation.set(false);
+ LOG.info("Unset host aggregator function for : " + instanceName);
+ break;
+ case CLUSTER:
+ performsClusterAggregation.set(false);
+ LOG.info("Unset cluster aggregator function for : " + instanceName);
+ }
+ }
+
+ /**
+ * Disconnect participant before controller shutdown
+ */
+ void stop() {
+ manager.disconnect();
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/availability/CheckpointManager.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/availability/CheckpointManager.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/availability/CheckpointManager.java
new file mode 100644
index 0000000..868fb93
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/availability/CheckpointManager.java
@@ -0,0 +1,95 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.metrics.core.timeline.availability;
+
+import org.I0Itec.zkclient.DataUpdater;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.helix.AccessOption;
+import org.apache.helix.ZNRecord;
+import org.apache.helix.store.zk.ZkHelixPropertyStore;
+import org.apache.zookeeper.data.Stat;
+
+public class CheckpointManager {
+ private final ZkHelixPropertyStore<ZNRecord> propertyStore;
+ private static final Log LOG = LogFactory.getLog(CheckpointManager.class);
+
+ static final String ZNODE_FIELD = "checkpoint";
+ static final String CHECKPOINT_PATH_PREFIX = "CHECKPOINTS";
+
+ public CheckpointManager(ZkHelixPropertyStore<ZNRecord> propertyStore) {
+ this.propertyStore = propertyStore;
+ }
+
+ /**
+ * Read aggregator checkpoint from zookeeper
+ *
+ * @return timestamp
+ */
+ public long readCheckpoint(AggregationTaskRunner.AGGREGATOR_NAME aggregatorName) {
+ String path = getCheckpointZKPath(aggregatorName);
+ LOG.debug("Reading checkpoint at " + path);
+ Stat stat = new Stat();
+ ZNRecord znRecord = propertyStore.get(path, stat, AccessOption.PERSISTENT);
+ if (LOG.isTraceEnabled()) {
+ LOG.trace("Stat => " + stat);
+ }
+ long checkpoint = znRecord != null ? znRecord.getLongField(ZNODE_FIELD, -1) : -1;
+ LOG.debug("Checkpoint value = " + checkpoint);
+ return checkpoint;
+ }
+
+ /**
+ * Write aggregator checkpoint in zookeeper
+ *
+ * @param value timestamp
+ * @return sucsess
+ */
+ public boolean writeCheckpoint(AggregationTaskRunner.AGGREGATOR_NAME aggregatorName, long value) {
+ String path = getCheckpointZKPath(aggregatorName);
+ LOG.debug(String.format("Saving checkpoint at %s with value %s", path, value));
+ return propertyStore.update(path, new CheckpointDataUpdater(path, value), AccessOption.PERSISTENT);
+ }
+
+ static class CheckpointDataUpdater implements DataUpdater<ZNRecord> {
+ final String path;
+ final Long value;
+
+ public CheckpointDataUpdater(String path, Long value) {
+ this.path = path;
+ this.value = value;
+ }
+
+ @Override
+ public ZNRecord update(ZNRecord currentData) {
+ if (currentData == null) {
+ currentData = new ZNRecord(path);
+ }
+ currentData.setLongField(ZNODE_FIELD, value);
+ return currentData;
+ }
+ }
+
+ String getCheckpointZKPath(AggregationTaskRunner.AGGREGATOR_NAME aggregatorName) {
+ StringBuilder sb = new StringBuilder("/");
+ sb.append(CHECKPOINT_PATH_PREFIX);
+ sb.append("/");
+ sb.append(AggregationTaskRunner.ACTUAL_AGGREGATOR_NAMES.get(aggregatorName));
+ return sb.toString();
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/availability/MetricCollectorHAController.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/availability/MetricCollectorHAController.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/availability/MetricCollectorHAController.java
new file mode 100644
index 0000000..ee28d87
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/availability/MetricCollectorHAController.java
@@ -0,0 +1,307 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.metrics.core.timeline.availability;
+
+import static org.apache.helix.model.IdealState.RebalanceMode.FULL_AUTO;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.TreeSet;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+
+import org.I0Itec.zkclient.exception.ZkNoNodeException;
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.lang.StringUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.ambari.metrics.core.timeline.MetricsSystemInitializationException;
+import org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguration;
+import org.apache.helix.HelixAdmin;
+import org.apache.helix.HelixException;
+import org.apache.helix.HelixManager;
+import org.apache.helix.HelixManagerFactory;
+import org.apache.helix.InstanceType;
+import org.apache.helix.NotificationContext;
+import org.apache.helix.controller.GenericHelixController;
+import org.apache.helix.manager.zk.ZKHelixAdmin;
+import org.apache.helix.model.ExternalView;
+import org.apache.helix.model.InstanceConfig;
+import org.apache.helix.model.LiveInstance;
+import org.apache.helix.model.OnlineOfflineSMD;
+import org.apache.helix.model.StateModelDefinition;
+import org.apache.helix.tools.StateModelConfigGenerator;
+
+import com.google.common.base.Joiner;
+
+;
+
+public class MetricCollectorHAController {
+ private static final Log LOG = LogFactory.getLog(MetricCollectorHAController.class);
+
+ static final String CLUSTER_NAME = "ambari-metrics-cluster";
+ static final String METRIC_AGGREGATORS = "METRIC_AGGREGATORS";
+ static final String DEFAULT_STATE_MODEL = OnlineOfflineSMD.name;
+ static final String INSTANCE_NAME_DELIMITER = "_";
+
+ final String zkConnectUrl;
+ final String instanceHostname;
+ final InstanceConfig instanceConfig;
+ final AggregationTaskRunner aggregationTaskRunner;
+ final TimelineMetricConfiguration configuration;
+
+ // Cache list of known live instances
+ final List<String> liveInstanceNames = new ArrayList<>();
+
+ // Helix Admin
+ HelixAdmin admin;
+ // Helix Manager
+ HelixManager manager;
+
+ private volatile boolean isInitialized = false;
+
+ public MetricCollectorHAController(TimelineMetricConfiguration configuration) {
+ this.configuration = configuration;
+ String instancePort;
+ try {
+ instanceHostname = configuration.getInstanceHostnameFromEnv();
+ instancePort = configuration.getInstancePort();
+
+ } catch (Exception e) {
+ LOG.error("Error reading configs from classpath, will resort to defaults.", e);
+ throw new MetricsSystemInitializationException(e.getMessage());
+ }
+
+ try {
+ String zkClientPort = configuration.getClusterZKClientPort();
+ String zkQuorum = configuration.getClusterZKQuorum();
+
+ if (StringUtils.isEmpty(zkClientPort) || StringUtils.isEmpty(zkQuorum)) {
+ throw new Exception("Unable to parse zookeeper quorum. clientPort = "
+ + zkClientPort +", quorum = " + zkQuorum);
+ }
+
+ zkConnectUrl = configuration.getZkConnectionUrl(zkClientPort, zkQuorum);
+
+ } catch (Exception e) {
+ LOG.error("Unable to load hbase-site from classpath.", e);
+ throw new MetricsSystemInitializationException(e.getMessage());
+ }
+
+ instanceConfig = new InstanceConfig(instanceHostname + INSTANCE_NAME_DELIMITER + instancePort);
+ instanceConfig.setHostName(instanceHostname);
+ instanceConfig.setPort(instancePort);
+ instanceConfig.setInstanceEnabled(true);
+ aggregationTaskRunner = new AggregationTaskRunner(
+ instanceConfig.getInstanceName(), zkConnectUrl, getClusterName());
+ }
+
+ /**
+ * Name of Helix znode
+ */
+ public String getClusterName() {
+ return CLUSTER_NAME;
+ }
+
+ /**
+ * Initialize the instance with zookeeper via Helix
+ */
+ public void initializeHAController() throws Exception {
+ String clusterName = getClusterName();
+ admin = new ZKHelixAdmin(zkConnectUrl);
+ // create cluster
+ LOG.info("Creating zookeeper cluster node: " + clusterName);
+ boolean clusterAdded = admin.addCluster(clusterName, false);
+ LOG.info("Was cluster added successfully? " + clusterAdded);
+
+ // Adding host to the cluster
+ boolean success = false;
+ int tries = 5;
+ int sleepTimeInSeconds = 5;
+
+ for (int i = 0; i < tries && !success; i++) {
+ try {
+ List<String> nodes = admin.getInstancesInCluster(clusterName);
+ if (CollectionUtils.isEmpty(nodes) || !nodes.contains(instanceConfig.getInstanceName())) {
+ LOG.info("Adding participant instance " + instanceConfig);
+ admin.addInstance(clusterName, instanceConfig);
+ }
+ success = true;
+ } catch (HelixException | ZkNoNodeException ex) {
+ LOG.warn("Helix Cluster not yet setup fully.");
+ if (i < tries - 1) {
+ LOG.info("Waiting for " + sleepTimeInSeconds + " seconds and retrying.");
+ TimeUnit.SECONDS.sleep(sleepTimeInSeconds);
+ } else {
+ LOG.error(ex);
+ }
+ }
+ }
+
+ if (!success) {
+ LOG.info("Trying to create " + clusterName + " again since waiting for the creation did not help.");
+ admin.addCluster(clusterName, true);
+ List<String> nodes = admin.getInstancesInCluster(clusterName);
+ if (CollectionUtils.isEmpty(nodes) || !nodes.contains(instanceConfig.getInstanceName())) {
+ LOG.info("Adding participant instance " + instanceConfig);
+ admin.addInstance(clusterName, instanceConfig);
+ }
+ }
+
+ // Add a state model
+ if (admin.getStateModelDef(clusterName, DEFAULT_STATE_MODEL) == null) {
+ LOG.info("Adding ONLINE-OFFLINE state model to the cluster");
+ admin.addStateModelDef(clusterName, DEFAULT_STATE_MODEL, new StateModelDefinition(
+ StateModelConfigGenerator.generateConfigForOnlineOffline()));
+ }
+
+ // Add resources with 1 cluster-wide replica
+ // Since our aggregators are unbalanced in terms of work distribution we
+ // only need to distribute writes to METRIC_AGGREGATE and
+ // METRIC_RECORD_MINUTE
+ List<String> resources = admin.getResourcesInCluster(clusterName);
+ if (!resources.contains(METRIC_AGGREGATORS)) {
+ LOG.info("Adding resource " + METRIC_AGGREGATORS + " with 2 partitions and 1 replicas");
+ admin.addResource(clusterName, METRIC_AGGREGATORS, 2, DEFAULT_STATE_MODEL, FULL_AUTO.toString());
+ }
+ // this will set up the ideal state, it calculates the preference list for
+ // each partition similar to consistent hashing
+ admin.rebalance(clusterName, METRIC_AGGREGATORS, 1);
+
+ // Start participant
+ startAggregators();
+
+ // Start controller
+ startController();
+
+ Runtime.getRuntime().addShutdownHook(new Thread() {
+ @Override
+ public void run() {
+ aggregationTaskRunner.stop();
+ manager.disconnect();
+ }
+ });
+
+ isInitialized = true;
+ }
+
+ /**
+ * Return true if HA controller is enabled.
+ */
+ public boolean isInitialized() {
+ return isInitialized;
+ }
+
+ private void startAggregators() {
+ try {
+ aggregationTaskRunner.initialize();
+
+ } catch (Exception e) {
+ LOG.error("Unable to start aggregators.", e);
+ throw new MetricsSystemInitializationException(e.getMessage());
+ }
+ }
+
+ private void startController() throws Exception {
+ manager = HelixManagerFactory.getZKHelixManager(
+ getClusterName(),
+ instanceHostname,
+ InstanceType.CONTROLLER,
+ zkConnectUrl
+ );
+
+ manager.connect();
+ HelixController controller = new HelixController();
+ manager.addLiveInstanceChangeListener(controller);
+ }
+
+ public AggregationTaskRunner getAggregationTaskRunner() {
+ return aggregationTaskRunner;
+ }
+
+ public List<String> getLiveInstanceHostNames() {
+ List<String> liveInstanceHostNames = new ArrayList<>();
+
+ for (String instance : liveInstanceNames) {
+ liveInstanceHostNames.add(instance.split(INSTANCE_NAME_DELIMITER)[0]);
+ }
+
+ return liveInstanceHostNames;
+ }
+
+ public class HelixController extends GenericHelixController {
+ ScheduledExecutorService executorService = Executors.newSingleThreadScheduledExecutor();
+ Joiner joiner = Joiner.on(", ").skipNulls();
+
+ @Override
+ public void onLiveInstanceChange(List<LiveInstance> liveInstances, NotificationContext changeContext) {
+ super.onLiveInstanceChange(liveInstances, changeContext);
+
+ liveInstanceNames.clear();
+ for (LiveInstance instance : liveInstances) {
+ liveInstanceNames.add(instance.getInstanceName());
+ }
+
+ LOG.info("Detected change in liveliness of Collector instances. " +
+ "LiveIsntances = " + joiner.join(liveInstanceNames));
+ // Print HA state - after some delay
+ executorService.schedule(new Runnable() {
+ @Override
+ public void run() {
+ printClusterState();
+ }
+ }, 30, TimeUnit.SECONDS);
+
+
+ }
+ }
+
+ public void printClusterState() {
+ StringBuilder sb = new StringBuilder("\n######################### Cluster HA state ########################");
+
+ ExternalView resourceExternalView = admin.getResourceExternalView(getClusterName(), METRIC_AGGREGATORS);
+ if (resourceExternalView != null) {
+ getPrintableResourceState(resourceExternalView, METRIC_AGGREGATORS, sb);
+ }
+ sb.append("\n##################################################");
+ LOG.info(sb.toString());
+ }
+
+ private void getPrintableResourceState(ExternalView resourceExternalView,
+ String resourceName,
+ StringBuilder sb) {
+ TreeSet<String> sortedSet = new TreeSet<>(resourceExternalView.getPartitionSet());
+ sb.append("\nCLUSTER: ");
+ sb.append(getClusterName());
+ sb.append("\nRESOURCE: ");
+ sb.append(resourceName);
+ for (String partitionName : sortedSet) {
+ sb.append("\nPARTITION: ");
+ sb.append(partitionName).append("\t");
+ Map<String, String> states = resourceExternalView.getStateMap(partitionName);
+ for (Map.Entry<String, String> stateEntry : states.entrySet()) {
+ sb.append("\t");
+ sb.append(stateEntry.getKey());
+ sb.append("\t");
+ sb.append(stateEntry.getValue());
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/availability/OnlineOfflineStateModelFactory.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/availability/OnlineOfflineStateModelFactory.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/availability/OnlineOfflineStateModelFactory.java
new file mode 100644
index 0000000..78a3199
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/availability/OnlineOfflineStateModelFactory.java
@@ -0,0 +1,69 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.metrics.core.timeline.availability;
+
+import static org.apache.ambari.metrics.core.timeline.availability.AggregationTaskRunner.PARTITION_AGGREGATION_TYPES;
+
+import org.apache.ambari.metrics.core.timeline.aggregators.TimelineMetricAggregator;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.helix.NotificationContext;
+import org.apache.helix.model.Message;
+import org.apache.helix.participant.statemachine.StateModel;
+import org.apache.helix.participant.statemachine.StateModelFactory;
+
+public class OnlineOfflineStateModelFactory extends StateModelFactory<StateModel> {
+ private static final Log LOG = LogFactory.getLog(OnlineOfflineStateModelFactory.class);
+ private final String instanceName;
+ private final AggregationTaskRunner taskRunner;
+
+ public OnlineOfflineStateModelFactory(String instanceName, AggregationTaskRunner taskRunner) {
+ this.instanceName = instanceName;
+ this.taskRunner = taskRunner;
+ }
+
+ @Override
+ public StateModel createNewStateModel(String resourceName, String partition) {
+ LOG.info("Received request to process partition = " + partition + ", for " +
+ "resource = " + resourceName + ", at " + instanceName);
+ return new OnlineOfflineStateModel();
+ }
+
+ public class OnlineOfflineStateModel extends StateModel {
+ public void onBecomeOnlineFromOffline(Message message, NotificationContext context) {
+ String partitionName = message.getPartitionName();
+ LOG.info("Received transition to Online from Offline for partition: " + partitionName);
+ TimelineMetricAggregator.AGGREGATOR_TYPE type = PARTITION_AGGREGATION_TYPES.get(partitionName);
+ taskRunner.setPartitionAggregationFunction(type);
+ }
+
+ public void onBecomeOfflineFromOnline(Message message, NotificationContext context) {
+ String partitionName = message.getPartitionName();
+ LOG.info("Received transition to Offline from Online for partition: " + partitionName);
+ TimelineMetricAggregator.AGGREGATOR_TYPE type = PARTITION_AGGREGATION_TYPES.get(partitionName);
+ taskRunner.unsetPartitionAggregationFunction(type);
+ }
+
+ public void onBecomeDroppedFromOffline(Message message, NotificationContext context) {
+ String partitionName = message.getPartitionName();
+ LOG.info("Received transition to Dropped from Offline for partition: " + partitionName);
+ TimelineMetricAggregator.AGGREGATOR_TYPE type = PARTITION_AGGREGATION_TYPES.get(partitionName);
+ taskRunner.unsetPartitionAggregationFunction(type);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/discovery/TimelineMetricHostMetadata.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/discovery/TimelineMetricHostMetadata.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/discovery/TimelineMetricHostMetadata.java
new file mode 100644
index 0000000..7645bd0
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/discovery/TimelineMetricHostMetadata.java
@@ -0,0 +1,60 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.metrics.core.timeline.discovery;
+
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+
+public class TimelineMetricHostMetadata {
+ //need concurrent data structure, only keys are used.
+ private ConcurrentHashMap<String, String> hostedApps = new ConcurrentHashMap<>();
+ private byte[] uuid;
+
+ // Default constructor
+ public TimelineMetricHostMetadata() {
+ }
+
+ public TimelineMetricHostMetadata(ConcurrentHashMap<String, String> hostedApps) {
+ this.hostedApps = hostedApps;
+ }
+
+ public TimelineMetricHostMetadata(Set<String> hostedApps) {
+ ConcurrentHashMap<String, String> appIdsMap = new ConcurrentHashMap<>();
+ for (String appId : hostedApps) {
+ appIdsMap.put(appId, appId);
+ }
+ this.hostedApps = appIdsMap;
+ }
+
+ public ConcurrentHashMap<String, String> getHostedApps() {
+ return hostedApps;
+ }
+
+ public void setHostedApps(ConcurrentHashMap<String, String> hostedApps) {
+ this.hostedApps = hostedApps;
+ }
+
+ public byte[] getUuid() {
+ return uuid;
+ }
+
+ public void setUuid(byte[] uuid) {
+ this.uuid = uuid;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/discovery/TimelineMetricMetadataKey.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/discovery/TimelineMetricMetadataKey.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/discovery/TimelineMetricMetadataKey.java
new file mode 100644
index 0000000..d308ce1
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/discovery/TimelineMetricMetadataKey.java
@@ -0,0 +1,80 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.metrics.core.timeline.discovery;
+
+import javax.xml.bind.annotation.XmlRootElement;
+
+import org.apache.commons.lang3.StringUtils;
+
+@XmlRootElement
+public class TimelineMetricMetadataKey {
+ String metricName;
+ String appId;
+ String instanceId;
+
+ public TimelineMetricMetadataKey(String metricName, String appId, String instanceId) {
+ this.metricName = metricName;
+ this.appId = appId;
+ this.instanceId = instanceId;
+ }
+
+ public String getMetricName() {
+ return metricName;
+ }
+
+ public String getAppId() {
+ return appId;
+ }
+
+ public String getInstanceId() {
+ return instanceId;
+ }
+
+ public void setAppId(String appId) {
+ this.appId = appId;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+
+ TimelineMetricMetadataKey that = (TimelineMetricMetadataKey) o;
+
+ if (!metricName.equals(that.metricName)) return false;
+ if (!appId.equals(that.appId)) return false;
+ return (StringUtils.isNotEmpty(instanceId) ? instanceId.equals(that.instanceId) : StringUtils.isEmpty(that.instanceId));
+ }
+
+ @Override
+ public int hashCode() {
+ int result = metricName.hashCode();
+ result = 31 * result + (appId != null ? appId.hashCode() : 0);
+ result = 31 * result + (instanceId != null ? instanceId.hashCode() : 0);
+ return result;
+ }
+
+ @Override
+ public String toString() {
+ return "TimelineMetricMetadataKey{" +
+ "metricName='" + metricName + '\'' +
+ ", appId='" + appId + '\'' +
+ ", instanceId='" + instanceId + '\'' +
+ '}';
+ }
+}
[03/28] ambari git commit: AMBARI-22740 : Fix integration test for
HBase in branch-3.0-ams due to UUID changes. (avijayan)
Posted by av...@apache.org.
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricStoreWatcherTest.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricStoreWatcherTest.java b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricStoreWatcherTest.java
deleted file mode 100644
index dd0378d..0000000
--- a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricStoreWatcherTest.java
+++ /dev/null
@@ -1,109 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline;
-
-import org.apache.hadoop.metrics2.sink.timeline.Precision;
-import org.apache.hadoop.metrics2.sink.timeline.TimelineMetric;
-import org.apache.hadoop.metrics2.sink.timeline.TimelineMetrics;
-import org.apache.hadoop.metrics2.sink.timeline.TopNConfig;
-import org.apache.hadoop.util.ExitUtil;
-import org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse;
-import org.easymock.EasyMock;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.powermock.core.classloader.annotations.PrepareForTest;
-import org.powermock.modules.junit4.PowerMockRunner;
-
-import java.util.List;
-
-import static org.easymock.EasyMock.anyObject;
-import static org.easymock.EasyMock.anyString;
-import static org.easymock.EasyMock.createNiceMock;
-import static org.easymock.EasyMock.eq;
-import static org.easymock.EasyMock.expect;
-import static org.easymock.EasyMock.expectLastCall;
-import static org.easymock.EasyMock.replay;
-import static org.easymock.EasyMock.verify;
-import static org.powermock.api.easymock.PowerMock.mockStatic;
-import static org.powermock.api.easymock.PowerMock.replayAll;
-import static org.powermock.api.easymock.PowerMock.verifyAll;
-
-@RunWith(PowerMockRunner.class)
-@PrepareForTest(ExitUtil.class)
-public class TimelineMetricStoreWatcherTest {
-
- @Test
- public void testRunPositive() throws Exception {
- TimelineMetricStore metricStore = createNiceMock(TimelineMetricStore.class);
-
- expect(metricStore.putMetrics(anyObject(TimelineMetrics.class)))
- .andReturn(new TimelinePutResponse());
-
- // metric found
- expect(metricStore.getTimelineMetrics(EasyMock.<List<String>>anyObject(),
- EasyMock.<List<String>>anyObject(), anyObject(String.class),
- anyObject(String.class), anyObject(Long.class), anyObject(Long.class),
- eq(Precision.SECONDS), eq(1), eq(true), anyObject(TopNConfig.class), anyString()))
- .andReturn(null).anyTimes();
-
- mockStatic(ExitUtil.class);
-
- replay(metricStore);
-
- TimelineMetricStoreWatcher timelineMetricStoreWatcher =
- new TimelineMetricStoreWatcher(metricStore, TimelineMetricConfiguration.getInstance());
- timelineMetricStoreWatcher.run();
- timelineMetricStoreWatcher.run();
- timelineMetricStoreWatcher.run();
- verify(metricStore);
-
- }
-
- @Test
- public void testRunNegative() throws Exception {
- TimelineMetricStore metricStore = createNiceMock(TimelineMetricStore.class);
-
- expect(metricStore.putMetrics(anyObject(TimelineMetrics.class)))
- .andReturn(new TimelinePutResponse());
-
- // no metrics found
- expect(metricStore.getTimelineMetrics(EasyMock.<List<String>>anyObject(),
- EasyMock.<List<String>>anyObject(), anyObject(String.class),
- anyObject(String.class), anyObject(Long.class), anyObject(Long.class),
- eq(Precision.SECONDS), eq(1), eq(true), anyObject(TopNConfig.class), anyString()))
- .andReturn(null).anyTimes();
-
- String msg = "Error getting metrics from TimelineMetricStore. " +
- "Shutting down by TimelineMetricStoreWatcher.";
- mockStatic(ExitUtil.class);
- ExitUtil.terminate(-1, msg);
- expectLastCall().anyTimes();
-
- replayAll();
-
- TimelineMetricStoreWatcher timelineMetricStoreWatcher =
- new TimelineMetricStoreWatcher(metricStore, TimelineMetricConfiguration.getInstance());
- timelineMetricStoreWatcher.run();
- timelineMetricStoreWatcher.run();
- timelineMetricStoreWatcher.run();
-
- verifyAll();
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricsAggregatorMemorySink.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricsAggregatorMemorySink.java b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricsAggregatorMemorySink.java
deleted file mode 100644
index 51cde4a..0000000
--- a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricsAggregatorMemorySink.java
+++ /dev/null
@@ -1,141 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline;
-
-import org.apache.hadoop.metrics2.sink.timeline.MetricClusterAggregate;
-import org.apache.hadoop.metrics2.sink.timeline.MetricHostAggregate;
-import org.apache.hadoop.metrics2.sink.timeline.Precision;
-import org.apache.hadoop.metrics2.sink.timeline.TimelineMetric;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.aggregators.TimelineClusterMetric;
-
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Map.Entry;
-
-/**
- * Aggregator Memory sink implementation to perform test
- */
-public class TimelineMetricsAggregatorMemorySink
- implements TimelineMetricsAggregatorSink {
-
- private static Map<Precision, Map<TimelineMetric, MetricHostAggregate>> hostAggregateRecords =
- new HashMap<>();
- private static Map<Precision, Map<TimelineClusterMetric, MetricHostAggregate>> clusterTimeAggregateRecords =
- new HashMap<>();
- private static Map<TimelineClusterMetric, MetricClusterAggregate> clusterAggregateRecords =
- new HashMap<>();
-
- @Override
- public void saveHostAggregateRecords(
- Map<TimelineMetric, MetricHostAggregate> hostAggregateMap,
- Precision precision) {
- if (hostAggregateMap == null || hostAggregateMap.size() == 0) {
- return;
- }
-
- Map<TimelineMetric, MetricHostAggregate> aggregatedValue = null;
- if (hostAggregateRecords.containsKey(precision)) {
- aggregatedValue = hostAggregateRecords.get(precision);
- } else {
- aggregatedValue = new HashMap<>();
- hostAggregateRecords.put(precision, aggregatedValue);
- }
-
- for (Entry<TimelineMetric, MetricHostAggregate> entry : hostAggregateMap
- .entrySet()) {
- TimelineMetric timelineMetricClone = new TimelineMetric(entry.getKey());
- MetricHostAggregate hostAggregate = entry.getValue();
- MetricHostAggregate hostAggregateClone = new MetricHostAggregate(
- hostAggregate.getSum(), (int) hostAggregate.getNumberOfSamples(),
- hostAggregate.getDeviation(), hostAggregate.getMax(),
- hostAggregate.getMin());
- aggregatedValue.put(timelineMetricClone, hostAggregateClone);
- }
- }
-
- @Override
- public void saveClusterTimeAggregateRecords(
- Map<TimelineClusterMetric, MetricHostAggregate> clusterTimeAggregateMap,
- Precision precision) {
- if (clusterTimeAggregateMap == null
- || clusterTimeAggregateMap.size() == 0) {
- return;
- }
-
- Map<TimelineClusterMetric, MetricHostAggregate> aggregatedValue = null;
- if (clusterTimeAggregateRecords.containsKey(precision)) {
- aggregatedValue = clusterTimeAggregateRecords.get(precision);
- } else {
- aggregatedValue = new HashMap<>();
- clusterTimeAggregateRecords.put(precision, aggregatedValue);
- }
-
- for (Entry<TimelineClusterMetric, MetricHostAggregate> entry : clusterTimeAggregateMap
- .entrySet()) {
- TimelineClusterMetric clusterMetric = entry.getKey();
- TimelineClusterMetric clusterMetricClone =
- new TimelineClusterMetric(clusterMetric.getMetricName(),
- clusterMetric.getAppId(), clusterMetric.getInstanceId(),
- clusterMetric.getTimestamp());
- MetricHostAggregate hostAggregate = entry.getValue();
- MetricHostAggregate hostAggregateClone = new MetricHostAggregate(
- hostAggregate.getSum(), (int) hostAggregate.getNumberOfSamples(),
- hostAggregate.getDeviation(), hostAggregate.getMax(),
- hostAggregate.getMin());
- aggregatedValue.put(clusterMetricClone, hostAggregateClone);
- }
- }
-
- @Override
- public void saveClusterAggregateRecords(
- Map<TimelineClusterMetric, MetricClusterAggregate> clusterAggregateMaps) {
-
- if (clusterAggregateMaps == null || clusterAggregateMaps.size() == 0) {
- return;
- }
-
- for (Entry<TimelineClusterMetric, MetricClusterAggregate> entry : clusterAggregateMaps
- .entrySet()) {
- TimelineClusterMetric clusterMetric = entry.getKey();
- TimelineClusterMetric clusterMetricClone =
- new TimelineClusterMetric(clusterMetric.getMetricName(),
- clusterMetric.getAppId(), clusterMetric.getInstanceId(),
- clusterMetric.getTimestamp());
- MetricClusterAggregate clusterAggregate = entry.getValue();
- MetricClusterAggregate clusterAggregateClone = new MetricClusterAggregate(
- clusterAggregate.getSum(), (int) clusterAggregate.getNumberOfHosts(),
- clusterAggregate.getDeviation(), clusterAggregate.getMax(),
- clusterAggregate.getMin());
- clusterAggregateRecords.put(clusterMetricClone, clusterAggregateClone);
- }
- }
-
- public Map<Precision, Map<TimelineMetric, MetricHostAggregate>> getHostAggregateRecords() {
- return Collections.unmodifiableMap(hostAggregateRecords);
- }
-
- public Map<Precision, Map<TimelineClusterMetric, MetricHostAggregate>> getClusterTimeAggregateRecords() {
- return Collections.unmodifiableMap(clusterTimeAggregateRecords);
- }
-
- public Map<TimelineClusterMetric, MetricClusterAggregate> getClusterAggregateRecords() {
- return Collections.unmodifiableMap(clusterAggregateRecords);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricsFilterTest.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricsFilterTest.java b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricsFilterTest.java
deleted file mode 100644
index a308248..0000000
--- a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricsFilterTest.java
+++ /dev/null
@@ -1,223 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline;
-
-import static org.easymock.EasyMock.expect;
-import static org.easymock.EasyMock.replay;
-
-import junit.framework.Assert;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.metrics2.sink.timeline.TimelineMetric;
-import org.easymock.EasyMock;
-import org.junit.Test;
-
-import java.net.URL;
-import java.util.HashSet;
-import java.util.Set;
-
-public class TimelineMetricsFilterTest {
-
- @Test
- public void testAppBlacklisting() throws Exception{
-
- Configuration metricsConf = new Configuration();
- metricsConf.set("timeline.metrics.apps.blacklist", "hbase,datanode,nimbus");
- TimelineMetricConfiguration configuration = EasyMock.createNiceMock(TimelineMetricConfiguration.class);
- expect(configuration.getMetricsConf()).andReturn(metricsConf).once();
- replay(configuration);
-
- TimelineMetricsFilter.initializeMetricFilter(configuration);
-
- TimelineMetric timelineMetric = new TimelineMetric();
-
- timelineMetric.setAppId("hbase");
- Assert.assertFalse(TimelineMetricsFilter.acceptMetric(timelineMetric));
-
- timelineMetric.setAppId("namenode");
- Assert.assertTrue(TimelineMetricsFilter.acceptMetric(timelineMetric));
-
- timelineMetric.setAppId("nimbus");
- Assert.assertFalse(TimelineMetricsFilter.acceptMetric(timelineMetric));
- }
-
- @Test
- public void testMetricWhitelisting() throws Exception {
-
- Configuration metricsConf = new Configuration();
- TimelineMetricConfiguration configuration = EasyMock.createNiceMock(TimelineMetricConfiguration.class);
- expect(configuration.getMetricsConf()).andReturn(metricsConf).once();
- expect(configuration.isWhitelistingEnabled()).andReturn(true).anyTimes();
- replay(configuration);
-
- URL fileUrl = ClassLoader.getSystemResource("test_data/metric_whitelist.dat");
-
- metricsConf.set("timeline.metrics.whitelist.file", fileUrl.getPath());
- TimelineMetricsFilter.initializeMetricFilter(configuration);
-
- TimelineMetric timelineMetric = new TimelineMetric();
-
- timelineMetric.setMetricName("cpu_system");
- Assert.assertTrue(TimelineMetricsFilter.acceptMetric(timelineMetric));
-
- timelineMetric.setMetricName("cpu_system1");
- Assert.assertFalse(TimelineMetricsFilter.acceptMetric(timelineMetric));
-
- timelineMetric.setMetricName("jvm.JvmMetrics.MemHeapUsedM");
- Assert.assertTrue(TimelineMetricsFilter.acceptMetric(timelineMetric));
-
- timelineMetric.setMetricName("dfs.FSNamesystem.TotalFiles");
- Assert.assertTrue(TimelineMetricsFilter.acceptMetric(timelineMetric));
- }
-
- @Test
- public void testTogether() throws Exception {
-
- Configuration metricsConf = new Configuration();
- metricsConf.set("timeline.metrics.apps.blacklist", "hbase,datanode,nimbus");
- TimelineMetricConfiguration configuration = EasyMock.createNiceMock(TimelineMetricConfiguration.class);
- expect(configuration.getMetricsConf()).andReturn(metricsConf).once();
- replay(configuration);
-
- URL fileUrl = ClassLoader.getSystemResource("test_data/metric_whitelist.dat");
- metricsConf.set("timeline.metrics.whitelist.file", fileUrl.getPath());
-
- TimelineMetricsFilter.initializeMetricFilter(configuration);
-
- TimelineMetric timelineMetric = new TimelineMetric();
-
- timelineMetric.setMetricName("cpu_system");
- timelineMetric.setAppId("hbase");
- Assert.assertFalse(TimelineMetricsFilter.acceptMetric(timelineMetric));
-
- timelineMetric.setMetricName("cpu_system");
- timelineMetric.setAppId("HOST");
- Assert.assertTrue(TimelineMetricsFilter.acceptMetric(timelineMetric));
-
- timelineMetric.setMetricName("jvm.JvmMetrics.MemHeapUsedM");
- Assert.assertTrue(TimelineMetricsFilter.acceptMetric(timelineMetric));
-
- timelineMetric.setMetricName("dfs.FSNamesystem.TotalFiles");
- Assert.assertTrue(TimelineMetricsFilter.acceptMetric(timelineMetric));
- }
-
- @Test
- public void testAmshbaseWhitelisting() throws Exception {
-
- TimelineMetricConfiguration configuration = EasyMock.createNiceMock(TimelineMetricConfiguration.class);
-
- Configuration metricsConf = new Configuration();
- expect(configuration.getMetricsConf()).andReturn(metricsConf).once();
-
- Set<String> whitelist = new HashSet();
- whitelist.add("regionserver.Server.Delete_99th_percentile");
- whitelist.add("regionserver.Server.Delete_max");
- whitelist.add("regionserver.Server.Delete_mean");
- expect(configuration.getAmshbaseWhitelist()).andReturn(whitelist).once();
-
- replay(configuration);
-
- TimelineMetricsFilter.initializeMetricFilter(configuration);
-
- TimelineMetric timelineMetric = new TimelineMetric();
-
- timelineMetric.setMetricName("regionserver.Server.Delete_max");
- timelineMetric.setAppId("ams-hbase");
- Assert.assertTrue(TimelineMetricsFilter.acceptMetric(timelineMetric));
-
- timelineMetric.setMetricName("regionserver.Server.Delete_min3333");
- timelineMetric.setAppId("ams-hbase");
- Assert.assertFalse(TimelineMetricsFilter.acceptMetric(timelineMetric));
-
- timelineMetric.setMetricName("jvm.JvmMetrics.MemHeapUsedM");
- timelineMetric.setAppId("hbase");
- Assert.assertTrue(TimelineMetricsFilter.acceptMetric(timelineMetric));
- }
-
- @Test
- public void testHybridFilter() throws Exception {
-
- // Whitelist Apps - namenode, nimbus
- // Blacklist Apps - datanode, kafka_broker
- // Accept ams-hbase whitelisting.
- // Reject non whitelisted metrics from non whitelisted Apps (Say hbase)
-
- TimelineMetricConfiguration configuration = EasyMock.createNiceMock(TimelineMetricConfiguration.class);
-
- Configuration metricsConf = new Configuration();
- metricsConf.set("timeline.metrics.apps.whitelist", "namenode,nimbus");
- metricsConf.set("timeline.metrics.apps.blacklist", "datanode,kafka_broker");
- URL fileUrl = ClassLoader.getSystemResource("test_data/metric_whitelist.dat");
- metricsConf.set("timeline.metrics.whitelist.file", fileUrl.getPath());
- expect(configuration.getMetricsConf()).andReturn(metricsConf).once();
-
- Set<String> whitelist = new HashSet();
- whitelist.add("regionserver.Server.Delete_99th_percentile");
- whitelist.add("regionserver.Server.Delete_max");
- whitelist.add("regionserver.Server.Delete_mean");
- expect(configuration.getAmshbaseWhitelist()).andReturn(whitelist).once();
-
- expect(configuration.isWhitelistingEnabled()).andReturn(true).anyTimes();
-
- replay(configuration);
-
- TimelineMetricsFilter.initializeMetricFilter(configuration);
-
- TimelineMetric timelineMetric = new TimelineMetric();
-
- //Test App Whitelisting
- timelineMetric.setMetricName("metric.a.b.c");
- timelineMetric.setAppId("namenode");
- Assert.assertTrue(TimelineMetricsFilter.acceptMetric(timelineMetric));
-
- timelineMetric.setMetricName("metric.d.e.f");
- timelineMetric.setAppId("nimbus");
- Assert.assertTrue(TimelineMetricsFilter.acceptMetric(timelineMetric));
-
- //Test App Blacklisting
- timelineMetric.setMetricName("metric.d.e.f");
- timelineMetric.setAppId("datanode");
- Assert.assertFalse(TimelineMetricsFilter.acceptMetric(timelineMetric));
-
- timelineMetric.setMetricName("metric.d.e.f");
- timelineMetric.setAppId("kafka_broker");
- Assert.assertFalse(TimelineMetricsFilter.acceptMetric(timelineMetric));
-
-
- //Test ams-hbase Whitelisting
- timelineMetric.setMetricName("regionserver.Server.Delete_max");
- timelineMetric.setAppId("ams-hbase");
- Assert.assertTrue(TimelineMetricsFilter.acceptMetric(timelineMetric));
-
- timelineMetric.setMetricName("regionserver.Server.Delete_min3333");
- timelineMetric.setAppId("ams-hbase");
- Assert.assertFalse(TimelineMetricsFilter.acceptMetric(timelineMetric));
-
- timelineMetric.setMetricName("regionserver.Server.Delete_mean");
- timelineMetric.setAppId("ams-hbase");
- Assert.assertTrue(TimelineMetricsFilter.acceptMetric(timelineMetric));
-
- //Test Metric Whitelisting
- timelineMetric.setMetricName("regionserver.WAL.SyncTime_max");
- timelineMetric.setAppId("hbase");
- Assert.assertTrue(TimelineMetricsFilter.acceptMetric(timelineMetric));
-
- timelineMetric.setMetricName("regionserver.WAL.metric.not.needed");
- timelineMetric.setAppId("hbase");
- Assert.assertFalse(TimelineMetricsFilter.acceptMetric(timelineMetric));
- }
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricsIgniteCacheTest.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricsIgniteCacheTest.java b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricsIgniteCacheTest.java
deleted file mode 100644
index 2cb66ba..0000000
--- a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricsIgniteCacheTest.java
+++ /dev/null
@@ -1,240 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline;
-
-
-import junit.framework.Assert;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.metrics2.sink.timeline.MetricClusterAggregate;
-import org.apache.hadoop.metrics2.sink.timeline.TimelineMetric;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.aggregators.TimelineClusterMetric;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.discovery.TimelineMetricMetadataManager;
-import org.junit.BeforeClass;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.powermock.core.classloader.annotations.PowerMockIgnore;
-import org.powermock.core.classloader.annotations.PrepareForTest;
-import org.powermock.modules.junit4.PowerMockRunner;
-
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.TreeMap;
-
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.CLUSTER_AGGREGATOR_APP_IDS;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.HOST_APP_ID;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.TIMELINE_METRICS_COLLECTOR_IGNITE_NODES;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.aggregators.AggregatorUtils.getRoundedCheckPointTimeMillis;
-import static org.easymock.EasyMock.anyObject;
-import static org.easymock.EasyMock.createNiceMock;
-import static org.easymock.EasyMock.expect;
-import static org.easymock.EasyMock.replay;
-import static org.powermock.api.easymock.PowerMock.mockStatic;
-import static org.powermock.api.easymock.PowerMock.replayAll;
-
-@RunWith(PowerMockRunner.class)
-@PrepareForTest(TimelineMetricConfiguration.class)
-
-@PowerMockIgnore("javax.management.*")
-public class TimelineMetricsIgniteCacheTest {
- private static TimelineMetricsIgniteCache timelineMetricsIgniteCache;
- @BeforeClass
- public static void setupConf() throws Exception {
- TimelineMetricConfiguration conf = new TimelineMetricConfiguration(new
- Configuration(), new Configuration());
- mockStatic(TimelineMetricConfiguration.class);
- expect(TimelineMetricConfiguration.getInstance()).andReturn(conf).anyTimes();
- conf.getMetricsConf().set(CLUSTER_AGGREGATOR_APP_IDS, "appIdForHostsAggr");
- conf.getMetricsConf().set(TIMELINE_METRICS_COLLECTOR_IGNITE_NODES, "localhost");
- replayAll();
-
- timelineMetricsIgniteCache = new TimelineMetricsIgniteCache();
- }
-
- @Test
- public void putEvictMetricsFromCacheSlicesMerging() throws Exception {
- long cacheSliceIntervalMillis = 30000L;
-
- TimelineMetricMetadataManager metricMetadataManagerMock = createNiceMock(TimelineMetricMetadataManager.class);
- expect(metricMetadataManagerMock.getUuid(anyObject(TimelineClusterMetric.class))).andReturn(new byte[16]).once();
- replay(metricMetadataManagerMock);
-
- long startTime = getRoundedCheckPointTimeMillis(System.currentTimeMillis(), cacheSliceIntervalMillis);
-
- long seconds = 1000;
- TreeMap<Long, Double> metricValues = new TreeMap<Long, Double>();
- /*
-
- 0 +30s +60s
- | | |
- (1)(2)(3) (4)(5)(6) h1
-
- */
- // Case 1 : data points are distributed equally, no values are lost, single host.
- metricValues.put(startTime + 4*seconds, 1.0);
- metricValues.put(startTime + 14*seconds, 2.0);
- metricValues.put(startTime + 24*seconds, 3.0);
- metricValues.put(startTime + 34*seconds, 4.0);
- metricValues.put(startTime + 44*seconds, 5.0);
- metricValues.put(startTime + 54*seconds, 6.0);
-
- TimelineMetric timelineMetric = new TimelineMetric("metric1", "host1", "app1", "instance1");
- timelineMetric.setStartTime(metricValues.firstKey());
- timelineMetric.addMetricValues(metricValues);
-
- Collection<TimelineMetric> timelineMetrics = new ArrayList<>();
- timelineMetrics.add(timelineMetric);
- timelineMetricsIgniteCache.putMetrics(timelineMetrics, metricMetadataManagerMock);
- Map<TimelineClusterMetric, MetricClusterAggregate> aggregateMap = timelineMetricsIgniteCache.evictMetricAggregates(startTime, startTime + 120*seconds);
-
- Assert.assertEquals(aggregateMap.size(), 2);
- TimelineClusterMetric timelineClusterMetric = new TimelineClusterMetric(timelineMetric.getMetricName(),
- timelineMetric.getAppId(), timelineMetric.getInstanceId(), startTime + 30*seconds);
-
- Assert.assertTrue(aggregateMap.containsKey(timelineClusterMetric));
- Assert.assertEquals(2.0, aggregateMap.get(timelineClusterMetric).getSum());
-
- timelineClusterMetric.setTimestamp(startTime + 2*30*seconds);
- Assert.assertTrue(aggregateMap.containsKey(timelineClusterMetric));
- Assert.assertEquals(5.0, aggregateMap.get(timelineClusterMetric).getSum());
-
- metricValues.clear();
- timelineMetrics.clear();
-
- /*
-
- 0 +30s +60s
- | | |
- (1)(2)(3) (4)(5)(6) h1, h2
-
- */
- // Case 2 : data points are distributed equally, no values are lost, two hosts.
- metricValues.put(startTime + 4*seconds, 1.0);
- metricValues.put(startTime + 14*seconds, 2.0);
- metricValues.put(startTime + 24*seconds, 3.0);
- metricValues.put(startTime + 34*seconds, 4.0);
- metricValues.put(startTime + 44*seconds, 5.0);
- metricValues.put(startTime + 54*seconds, 6.0);
-
- timelineMetric = new TimelineMetric("metric1", "host1", "app1", "instance1");
- timelineMetric.setMetricValues(metricValues);
-
- metricValues = new TreeMap<>();
- metricValues.put(startTime + 5*seconds, 2.0);
- metricValues.put(startTime + 15*seconds, 4.0);
- metricValues.put(startTime + 25*seconds, 6.0);
- metricValues.put(startTime + 35*seconds, 8.0);
- metricValues.put(startTime + 45*seconds, 10.0);
- metricValues.put(startTime + 55*seconds, 12.0);
- TimelineMetric timelineMetric2 = new TimelineMetric("metric1", "host2", "app1", "instance1");
- timelineMetric2.setMetricValues(metricValues);
-
- timelineMetrics = new ArrayList<>();
- timelineMetrics.add(timelineMetric);
- timelineMetrics.add(timelineMetric2);
- timelineMetricsIgniteCache.putMetrics(timelineMetrics, metricMetadataManagerMock);
- aggregateMap = timelineMetricsIgniteCache.evictMetricAggregates(startTime, startTime + 120*seconds);
-
- Assert.assertEquals(aggregateMap.size(), 2);
- timelineClusterMetric = new TimelineClusterMetric(timelineMetric.getMetricName(),
- timelineMetric.getAppId(), timelineMetric.getInstanceId(), startTime + 30*seconds);
-
- Assert.assertTrue(aggregateMap.containsKey(timelineClusterMetric));
- Assert.assertEquals(6.0, aggregateMap.get(timelineClusterMetric).getSum());
-
- timelineClusterMetric.setTimestamp(startTime + 2*30*seconds);
- Assert.assertTrue(aggregateMap.containsKey(timelineClusterMetric));
- Assert.assertEquals(15.0, aggregateMap.get(timelineClusterMetric).getSum());
-
- metricValues.clear();
- timelineMetrics.clear();
-
- Assert.assertEquals(0d, timelineMetricsIgniteCache.getPointInTimeCacheMetrics().get("Cluster_KeySize"));
- }
-
- @Test
- public void updateAppAggregatesFromHostMetricTest() {
- //make sure hosts metrics are aggregated for appIds from "timeline.metrics.service.cluster.aggregator.appIds"
-
- long cacheSliceIntervalMillis = 30000L;
-
- TimelineMetricMetadataManager metricMetadataManagerMock = createNiceMock(TimelineMetricMetadataManager.class);
- expect(metricMetadataManagerMock.getUuid(anyObject(TimelineClusterMetric.class))).andReturn(new byte[16]).once();
- expect(metricMetadataManagerMock.getHostedAppsCache()).andReturn(new HashMap<>()).anyTimes();
- replay(metricMetadataManagerMock);
-
- long startTime = getRoundedCheckPointTimeMillis(System.currentTimeMillis(), cacheSliceIntervalMillis);
-
- long seconds = 1000;
-
- TreeMap<Long, Double> metricValues = new TreeMap<Long, Double>();
- List<TimelineMetric> timelineMetrics = new ArrayList<>();
- TimelineMetric timelineMetric;
-
- metricValues = new TreeMap<>();
- metricValues.put(startTime + 15*seconds, 1.0);
- metricValues.put(startTime + 55*seconds, 2.0);
- timelineMetric = new TimelineMetric("host_metric", "host1", HOST_APP_ID, "instance1");
- timelineMetric.setMetricValues(metricValues);
- timelineMetrics.add(timelineMetric);
-
- metricValues = new TreeMap<>();
- metricValues.put(startTime + 45*seconds, 3.0);
- metricValues.put(startTime + 85*seconds, 4.0);
- timelineMetric = new TimelineMetric("app_metric", "host1", "appIdForHostsAggr", "instance1");
- timelineMetric.setMetricValues(metricValues);
- timelineMetrics.add(timelineMetric);
-
- metricValues = new TreeMap<>();
- metricValues.put(startTime + 85*seconds, 5.0);
- timelineMetric = new TimelineMetric("host_metric", "host1", HOST_APP_ID, "instance1");
- timelineMetric.setMetricValues(metricValues);
- timelineMetrics.add(timelineMetric);
-
- metricValues = new TreeMap<>();
- metricValues.put(startTime + 85*seconds, 6.0);
- timelineMetric = new TimelineMetric("host_metric", "host2", HOST_APP_ID, "instance1");
- timelineMetric.setMetricValues(metricValues);
- timelineMetrics.add(timelineMetric);
-
- timelineMetricsIgniteCache.putMetrics(timelineMetrics, metricMetadataManagerMock);
-
- Map<TimelineClusterMetric, MetricClusterAggregate> aggregateMap = timelineMetricsIgniteCache.evictMetricAggregates(startTime, startTime + 120*seconds);
- Assert.assertEquals(aggregateMap.size(), 6);
- TimelineClusterMetric timelineClusterMetric = new TimelineClusterMetric(timelineMetric.getMetricName(),
- timelineMetric.getAppId(), timelineMetric.getInstanceId(), startTime + 90*seconds);
-
- Assert.assertTrue(aggregateMap.containsKey(timelineClusterMetric));
- Assert.assertEquals(11.0, aggregateMap.get(timelineClusterMetric).getSum());
-
- timelineClusterMetric = new TimelineClusterMetric("app_metric",
- "appIdForHostsAggr", "instance1", startTime + 90*seconds);
- Assert.assertTrue(aggregateMap.containsKey(timelineClusterMetric));
- Assert.assertEquals(4.0, aggregateMap.get(timelineClusterMetric).getSum());
-
- timelineClusterMetric = new TimelineClusterMetric("host_metric",
- "appIdForHostsAggr", "instance1", startTime + 90*seconds);
- Assert.assertTrue(aggregateMap.containsKey(timelineClusterMetric));
- Assert.assertEquals(5.0, aggregateMap.get(timelineClusterMetric).getSum());
-
- Assert.assertEquals(0d, timelineMetricsIgniteCache.getPointInTimeCacheMetrics().get("Cluster_KeySize"));
- }
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TopNConditionTest.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TopNConditionTest.java b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TopNConditionTest.java
deleted file mode 100644
index 5110191..0000000
--- a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TopNConditionTest.java
+++ /dev/null
@@ -1,105 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline;
-
-import junit.framework.Assert;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query.TopNCondition;
-import org.junit.Test;
-
-import java.util.ArrayList;
-import java.util.List;
-
-public class TopNConditionTest {
-
-
- @Test
- public void testTopNCondition() {
-
- List<String> metricNames = new ArrayList<>();
- List<String> hostnames = new ArrayList<>();
-
- //Valid Cases
-
- // "H" hosts and 1 Metric
- hostnames.add("h1");
- hostnames.add("h2");
- metricNames.add("m1");
- Assert.assertTrue(TopNCondition.isTopNHostCondition(metricNames, hostnames));
- Assert.assertFalse(TopNCondition.isTopNMetricCondition(metricNames, hostnames));
- hostnames.clear();
-
- // Host(s) with wildcard & 1 Metric
- hostnames.add("h%");
- hostnames.add("g1");
- Assert.assertTrue(TopNCondition.isTopNHostCondition(metricNames, hostnames));
- Assert.assertFalse(TopNCondition.isTopNMetricCondition(metricNames, hostnames));
- hostnames.clear();
-
- // M Metrics and No host
- metricNames.add("m2");
- metricNames.add("m3");
- Assert.assertFalse(TopNCondition.isTopNHostCondition(metricNames, hostnames));
- Assert.assertTrue(TopNCondition.isTopNMetricCondition(metricNames, hostnames));
-
- // M Metrics with wildcard and No host
- metricNames.add("m2");
- metricNames.add("m%");
- Assert.assertFalse(TopNCondition.isTopNHostCondition(metricNames, hostnames));
- Assert.assertTrue(TopNCondition.isTopNMetricCondition(metricNames, hostnames));
-
- // M Metrics with wildcard and 1 host
- metricNames.add("m2");
- metricNames.add("m%");
- hostnames.add("h1");
- Assert.assertFalse(TopNCondition.isTopNHostCondition(metricNames, hostnames));
- Assert.assertTrue(TopNCondition.isTopNMetricCondition(metricNames, hostnames));
-
- metricNames.clear();
- hostnames.clear();
-
- //Invalid Cases
- // M metrics and H hosts
- metricNames.add("m1");
- metricNames.add("m2");
- hostnames.add("h1");
- hostnames.add("h2");
- Assert.assertFalse(TopNCondition.isTopNHostCondition(metricNames, hostnames));
- Assert.assertFalse(TopNCondition.isTopNMetricCondition(metricNames, hostnames));
-
- metricNames.clear();
- hostnames.clear();
-
- // Wildcard in 1 and multiple in other
- metricNames.add("m1");
- metricNames.add("m2");
- hostnames.add("%");
- Assert.assertFalse(TopNCondition.isTopNHostCondition(metricNames, hostnames));
- Assert.assertFalse(TopNCondition.isTopNMetricCondition(metricNames, hostnames));
-
- metricNames.clear();
- hostnames.clear();
-
- //Wildcard in both
- metricNames.add("m%");
- hostnames.add("%");
- Assert.assertFalse(TopNCondition.isTopNHostCondition(metricNames, hostnames));
- Assert.assertFalse(TopNCondition.isTopNMetricCondition(metricNames, hostnames));
-
- }
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/AbstractTimelineAggregatorTest.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/AbstractTimelineAggregatorTest.java b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/AbstractTimelineAggregatorTest.java
deleted file mode 100644
index b4d0f0a..0000000
--- a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/AbstractTimelineAggregatorTest.java
+++ /dev/null
@@ -1,181 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.aggregators;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.availability.AggregationTaskRunner;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query.Condition;
-import org.junit.Before;
-import org.junit.Test;
-import java.io.IOException;
-import java.sql.ResultSet;
-import java.sql.SQLException;
-import java.util.concurrent.atomic.AtomicLong;
-
-import static junit.framework.Assert.assertEquals;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.AGGREGATOR_CHECKPOINT_DELAY;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.RESULTSET_FETCH_SIZE;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.aggregators.AggregatorUtils.getRoundedAggregateTimeMillis;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.aggregators.AggregatorUtils.getRoundedCheckPointTimeMillis;
-
-public class AbstractTimelineAggregatorTest {
-
- private AbstractTimelineAggregator agg;
-
- AtomicLong startTimeInDoWork;
- AtomicLong endTimeInDoWork;
- AtomicLong checkPoint;
- int actualRuns;
-
- long sleepIntervalMillis;
- int checkpointCutOffMultiplier;
-
- @Before
- public void setUp() throws Exception {
- sleepIntervalMillis = 5 * 60 * 1000l; //5 minutes
- checkpointCutOffMultiplier = 2;
-
- Configuration metricsConf = new Configuration();
- metricsConf.setInt(AGGREGATOR_CHECKPOINT_DELAY, 0);
- metricsConf.setInt(RESULTSET_FETCH_SIZE, 2000);
-
- startTimeInDoWork = new AtomicLong(0);
- endTimeInDoWork = new AtomicLong(0);
- checkPoint = new AtomicLong(-1);
- actualRuns = 0;
-
- agg = new AbstractTimelineAggregator(AggregationTaskRunner.AGGREGATOR_NAME.METRIC_AGGREGATE_SECOND, null, metricsConf) {
- @Override
- public boolean doWork(long startTime, long endTime) {
- startTimeInDoWork.set(startTime);
- endTimeInDoWork.set(endTime);
- actualRuns++;
-
- return true;
- }
-
- @Override
- protected Condition
- prepareMetricQueryCondition(long startTime, long endTime) {
- return null;
- }
-
- @Override
- protected void aggregate(ResultSet rs, long startTime,
- long endTime) throws IOException, SQLException {
- }
-
- @Override
- public Long getSleepIntervalMillis() {
- return sleepIntervalMillis;
- }
-
- @Override
- protected Integer getCheckpointCutOffMultiplier() {
- return checkpointCutOffMultiplier;
- }
-
- @Override
- public boolean isDisabled() {
- return false;
- }
-
- @Override
- protected String getCheckpointLocation() {
- return "dummy_ckptFile";
- }
-
- protected long readCheckPoint() {
- return checkPoint.get();
- }
-
- @Override
- protected void saveCheckPoint(long checkpointTime) throws IOException {
- checkPoint.set(checkpointTime);
- }
- };
-
- }
-
- @Test
- public void testDoWorkOnZeroDelay() throws Exception {
-
- long currentTime = System.currentTimeMillis();
- long roundedOffAggregatorTime = getRoundedCheckPointTimeMillis(currentTime,
- sleepIntervalMillis);
-
- //Test first run of aggregator with no checkpoint
- checkPoint.set(-1);
- agg.runOnce(sleepIntervalMillis);
- assertEquals("startTime should be zero", 0, startTimeInDoWork.get());
- assertEquals("endTime should be zero", 0, endTimeInDoWork.get());
- assertEquals(roundedOffAggregatorTime, checkPoint.get());
- assertEquals("Do not aggregate on first run", 0, actualRuns);
-
-// //Test first run with too "recent" checkpoint
- currentTime = System.currentTimeMillis();
- checkPoint.set(currentTime);
- agg.setSleepIntervalMillis(sleepIntervalMillis);
- agg.runOnce(sleepIntervalMillis);
- assertEquals("startTime should be zero", 0, startTimeInDoWork.get());
- assertEquals("endTime should be zero", 0, endTimeInDoWork.get());
- assertEquals("Do not aggregate on first run", 0, actualRuns);
-
- //Test first run with Too Old checkpoint
- currentTime = System.currentTimeMillis();
- checkPoint.set(currentTime - 16*60*1000); //Old checkpoint
- agg.runOnce(sleepIntervalMillis);
- long checkPointTime = getRoundedAggregateTimeMillis(sleepIntervalMillis);
- assertEquals("startTime should be zero", checkPointTime - sleepIntervalMillis, startTimeInDoWork.get());
- assertEquals("endTime should be zero", checkPointTime, endTimeInDoWork.get());
- assertEquals(roundedOffAggregatorTime, checkPoint.get());
- assertEquals("Do not aggregate on first run", 1, actualRuns);
-
-
-// //Test first run with perfect checkpoint (sleepIntervalMillis back)
- currentTime = System.currentTimeMillis();
- roundedOffAggregatorTime = getRoundedCheckPointTimeMillis(currentTime,
- sleepIntervalMillis);
- checkPointTime = roundedOffAggregatorTime - sleepIntervalMillis;
- long expectedCheckPoint = getRoundedCheckPointTimeMillis(checkPointTime, sleepIntervalMillis);
- checkPoint.set(checkPointTime);
- agg.runOnce(sleepIntervalMillis);
- assertEquals("startTime should the lower rounded time of the checkpoint time",
- expectedCheckPoint, startTimeInDoWork.get());
- assertEquals("endTime should the lower rounded time of the checkpoint time + sleepIntervalMillis",
- expectedCheckPoint + sleepIntervalMillis, endTimeInDoWork.get());
- assertEquals(expectedCheckPoint + sleepIntervalMillis,
- checkPoint.get());
- assertEquals("Aggregate on first run", 2, actualRuns);
-
- //Test edge case for checkpoint (2 x sleepIntervalMillis)
- currentTime = System.currentTimeMillis();
- checkPoint.set(currentTime - 2*sleepIntervalMillis + 5000);
- agg.runOnce(sleepIntervalMillis);
- long expectedStartTime = getRoundedCheckPointTimeMillis(currentTime - 2*sleepIntervalMillis + 5000, sleepIntervalMillis);
- assertEquals("startTime should the lower rounded time of the checkpoint time",
- expectedStartTime, startTimeInDoWork.get());
- assertEquals("startTime should the lower rounded time of the checkpoint time + sleepIntervalMillis",
- expectedStartTime + sleepIntervalMillis, endTimeInDoWork.get());
- assertEquals(expectedStartTime + sleepIntervalMillis,
- checkPoint.get());
- assertEquals("Aggregate on second run", 3, actualRuns);
-
-
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/DownSamplerTest.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/DownSamplerTest.java b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/DownSamplerTest.java
deleted file mode 100644
index 0fca2e3..0000000
--- a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/DownSamplerTest.java
+++ /dev/null
@@ -1,97 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.aggregators;
-
-import junit.framework.Assert;
-import org.apache.hadoop.conf.Configuration;
-import org.junit.Ignore;
-import org.junit.Test;
-
-import java.util.List;
-import java.util.Map;
-
-public class DownSamplerTest {
-
- @Test
- public void testGetDownsampleMetricPatterns() throws Exception {
-
- Configuration configuration = new Configuration();
- configuration.setIfUnset("timeline.metrics.downsampler.topn.metric.patterns", "pattern1,pattern2");
- configuration.setIfUnset("timeline.metrics.downsampler.lastvalue.metric.patterns", "pattern3");
-
- List<String> patterns = DownSamplerUtils.getDownsampleMetricPatterns(configuration);
- Assert.assertEquals(patterns.size(), 3);
- Assert.assertTrue(patterns.contains("pattern1"));
- Assert.assertTrue(patterns.contains("pattern2"));
- Assert.assertTrue(patterns.contains("pattern3"));
-
- Configuration configuration2 = new Configuration();
- patterns = DownSamplerUtils.getDownsampleMetricPatterns(configuration2);
- Assert.assertEquals(patterns.size(), 0);
- }
-
- @Test
- public void testGetDownSamplers() throws Exception {
-
- Configuration configuration = new Configuration();
- configuration.setIfUnset("timeline.metrics.downsampler.topn.metric.patterns", "pattern1,pattern2");
- configuration.setIfUnset("timeline.metrics.downsampler.test.metric.patterns", "pattern3");
-
- List<CustomDownSampler> downSamplers = DownSamplerUtils.getDownSamplers(configuration);
- Assert.assertEquals(downSamplers.size(), 1);
- Assert.assertTrue(downSamplers.get(0) instanceof TopNDownSampler);
- }
-
- @Ignore
- @Test
- public void testPrepareTopNDownSamplingStatement() throws Exception {
- Configuration configuration = new Configuration();
- configuration.setIfUnset("timeline.metrics.downsampler.topn.metric.patterns", "pattern1,pattern2");
- configuration.setIfUnset("timeline.metrics.downsampler.topn.value", "3");
-
- Map<String, String> conf = configuration.getValByRegex(DownSamplerUtils.downSamplerConfigPrefix);
-
- TopNDownSampler topNDownSampler = TopNDownSampler.fromConfig(conf);
- List<String> stmts = topNDownSampler.prepareDownSamplingStatement(14000000l, 14100000l, "METRIC_RECORD");
- Assert.assertEquals(stmts.size(),2);
- Assert.assertTrue(stmts.contains("SELECT METRIC_NAME, HOSTNAME, APP_ID, INSTANCE_ID, 14100000 AS SERVER_TIME, UNITS, " +
- "MAX(METRIC_MAX), 1, MAX(METRIC_MAX), MAX(METRIC_MAX) FROM METRIC_RECORD WHERE " +
- "METRIC_NAME LIKE 'pattern1' AND SERVER_TIME > 14000000 AND SERVER_TIME <= 14100000 " +
- "GROUP BY METRIC_NAME, HOSTNAME, APP_ID, INSTANCE_ID, UNITS ORDER BY MAX(METRIC_MAX) DESC LIMIT 3"));
-
- Assert.assertTrue(stmts.contains("SELECT METRIC_NAME, HOSTNAME, APP_ID, INSTANCE_ID, 14100000 AS SERVER_TIME, UNITS, " +
- "MAX(METRIC_MAX), 1, MAX(METRIC_MAX), MAX(METRIC_MAX) FROM METRIC_RECORD WHERE " +
- "METRIC_NAME LIKE 'pattern2' AND SERVER_TIME > 14000000 AND SERVER_TIME <= 14100000 " +
- "GROUP BY METRIC_NAME, HOSTNAME, APP_ID, INSTANCE_ID, UNITS ORDER BY MAX(METRIC_MAX) DESC LIMIT 3"));
-
- configuration.clear();
- configuration.setIfUnset("timeline.metrics.downsampler.topn.metric.patterns", "pattern1");
- configuration.setIfUnset("timeline.metrics.downsampler.topn.value", "4");
- configuration.setIfUnset("timeline.metrics.downsampler.topn.function", "sum");
- conf = configuration.getValByRegex(DownSamplerUtils.downSamplerConfigPrefix);
- topNDownSampler = TopNDownSampler.fromConfig(conf);
- stmts = topNDownSampler.prepareDownSamplingStatement(14000000l, 14100000l, "METRIC_AGGREGATE_MINUTE");
- Assert.assertEquals(stmts.size(),1);
-
- Assert.assertTrue(stmts.contains("SELECT METRIC_NAME, APP_ID, INSTANCE_ID, 14100000 AS SERVER_TIME, UNITS, " +
- "SUM(METRIC_SUM), 1, SUM(METRIC_SUM), SUM(METRIC_SUM) FROM METRIC_AGGREGATE_MINUTE WHERE " +
- "METRIC_NAME LIKE 'pattern1' AND SERVER_TIME > 14000000 AND SERVER_TIME <= 14100000 " +
- "GROUP BY METRIC_NAME, APP_ID, INSTANCE_ID, UNITS ORDER BY SUM(METRIC_SUM) DESC LIMIT 4"));
- }
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/ITClusterAggregator.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/ITClusterAggregator.java b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/ITClusterAggregator.java
deleted file mode 100644
index 1c5f41f..0000000
--- a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/ITClusterAggregator.java
+++ /dev/null
@@ -1,669 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.aggregators;
-
-
-import static junit.framework.Assert.assertEquals;
-import static junit.framework.Assert.assertNotNull;
-import static junit.framework.Assert.assertTrue;
-import static junit.framework.Assert.fail;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.MetricTestHelper.createEmptyTimelineClusterMetric;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.MetricTestHelper.prepareSingleTimelineMetric;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.CLUSTER_AGGREGATOR_APP_IDS;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query.PhoenixTransactSQL.GET_CLUSTER_AGGREGATE_SQL;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query.PhoenixTransactSQL.METRICS_CLUSTER_AGGREGATE_HOURLY_TABLE_NAME;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query.PhoenixTransactSQL.METRICS_CLUSTER_AGGREGATE_TABLE_NAME;
-
-import java.sql.Connection;
-import java.sql.PreparedStatement;
-import java.sql.ResultSet;
-import java.sql.SQLException;
-import java.sql.Statement;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.TreeMap;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.metrics2.sink.timeline.MetricClusterAggregate;
-import org.apache.hadoop.metrics2.sink.timeline.MetricHostAggregate;
-import org.apache.hadoop.metrics2.sink.timeline.TimelineMetric;
-import org.apache.hadoop.metrics2.sink.timeline.TimelineMetrics;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.AbstractMiniHBaseClusterTest;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.MetricTestHelper;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.discovery.TimelineMetricMetadataManager;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query.Condition;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query.DefaultCondition;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query.PhoenixTransactSQL;
-import org.junit.Test;
-
-import junit.framework.Assert;
-
-public class ITClusterAggregator extends AbstractMiniHBaseClusterTest {
- private final TimelineMetricReadHelper metricReader = new TimelineMetricReadHelper(false);
-
- private Configuration getConfigurationForTest(boolean useGroupByAggregators) {
- Configuration configuration = new Configuration();
- configuration.set("timeline.metrics.service.use.groupBy.aggregators", String.valueOf(useGroupByAggregators));
- return configuration;
- }
-
- @Test
- public void testShouldAggregateClusterProperly() throws Exception {
- // GIVEN
- TimelineMetricAggregator agg =
- TimelineMetricAggregatorFactory.createTimelineClusterAggregatorSecond(hdb,
- getConfigurationForTest(false), new TimelineMetricMetadataManager(new Configuration(), hdb), null, null);
- TimelineMetricReadHelper readHelper = new TimelineMetricReadHelper(false);
-
- long startTime = System.currentTimeMillis();
- long ctime = startTime;
- long minute = 60 * 1000;
- hdb.insertMetricRecords(prepareSingleTimelineMetric(ctime, "local1",
- "disk_free", 1));
- hdb.insertMetricRecords(prepareSingleTimelineMetric(ctime, "local2",
- "disk_free", 2));
- ctime += 2*minute;
- hdb.insertMetricRecords(prepareSingleTimelineMetric(ctime, "local1",
- "disk_free", 2));
- hdb.insertMetricRecords(prepareSingleTimelineMetric(ctime, "local2",
- "disk_free", 1));
-
- // WHEN
- long endTime = ctime + minute;
- boolean success = agg.doWork(startTime, endTime);
-
- //THEN
- Condition condition = new DefaultCondition(null, null, null, null, startTime,
- endTime, null, null, true);
- condition.setStatement(String.format(GET_CLUSTER_AGGREGATE_SQL,
- METRICS_CLUSTER_AGGREGATE_TABLE_NAME));
-
- PreparedStatement pstmt = PhoenixTransactSQL.prepareGetMetricsSqlStmt(conn, condition);
- ResultSet rs = pstmt.executeQuery();
-
- int recordCount = 0;
- while (rs.next()) {
- TimelineClusterMetric currentMetric = metricReader.fromResultSet(rs);
- MetricClusterAggregate currentHostAggregate =
- readHelper.getMetricClusterAggregateFromResultSet(rs);
-
- if ("disk_free".equals(currentMetric.getMetricName())) {
- assertEquals(2, currentHostAggregate.getNumberOfHosts());
- assertEquals(2.0, currentHostAggregate.getMax());
- assertEquals(1.0, currentHostAggregate.getMin());
- assertEquals(3.0, currentHostAggregate.getSum());
- recordCount++;
- } else {
- fail("Unexpected entry");
- }
- }
- }
-
- @Test
- public void testShouldAggregateClusterIgnoringInstance() throws Exception {
- // GIVEN
- TimelineMetricAggregator agg =
- TimelineMetricAggregatorFactory.createTimelineClusterAggregatorSecond(hdb,
- getConfigurationForTest(false), new TimelineMetricMetadataManager(new Configuration(), hdb), null, null);
- TimelineMetricReadHelper readHelper = new TimelineMetricReadHelper(false);
-
- long startTime = System.currentTimeMillis();
- long ctime = startTime;
- long minute = 60 * 1000 * 2;
-
- /**
- * Here we have two nodes with two instances each:
- * | local1 | local2 |
- * instance i1 | 1 | 2 |
- * instance i2 | 3 | 4 |
- *
- */
- // Four 1's at ctime - 100
- hdb.insertMetricRecords(prepareSingleTimelineMetric(ctime - 100, "local1",
- "i1", "disk_free", 1));
- // Four 2's at ctime - 100: different host
- hdb.insertMetricRecords(prepareSingleTimelineMetric(ctime - 100, "local2",
- "i1", "disk_free", 2));
- // Avoid overwrite
- hdb.insertMetricRecords(prepareSingleTimelineMetric(ctime + 100, "local1",
- "i2", "disk_free", 3));
- hdb.insertMetricRecords(prepareSingleTimelineMetric(ctime + 100, "local2",
- "i2", "disk_free", 4));
-
- ctime += minute;
-
- // Four 1's at ctime + 2 min
- hdb.insertMetricRecords(prepareSingleTimelineMetric(ctime - 100, "local1",
- "i1", "disk_free", 1));
- // Four 1's at ctime + 2 min - different host
- hdb.insertMetricRecords(prepareSingleTimelineMetric(ctime - 100, "local2",
- "i1", "disk_free", 3));
- hdb.insertMetricRecords(prepareSingleTimelineMetric(ctime + 100, "local1",
- "i2", "disk_free", 2));
- hdb.insertMetricRecords(prepareSingleTimelineMetric(ctime + 100, "local2",
- "i2", "disk_free", 4));
- // WHEN
- long endTime = ctime + minute;
- boolean success = agg.doWork(startTime - 1000, endTime + 1000);
-
- //THEN
- Condition condition = new DefaultCondition(null, null, null, null, startTime,
- endTime, null, null, true);
- condition.setStatement(String.format(GET_CLUSTER_AGGREGATE_SQL,
- METRICS_CLUSTER_AGGREGATE_TABLE_NAME));
-
- PreparedStatement pstmt = PhoenixTransactSQL.prepareGetMetricsSqlStmt(conn, condition);
- ResultSet rs = pstmt.executeQuery();
-
- int recordCount = 0;
- while (rs.next()) {
- TimelineClusterMetric currentMetric = metricReader.fromResultSet(rs);
- MetricClusterAggregate currentHostAggregate =
- readHelper.getMetricClusterAggregateFromResultSet(rs);
-
- if ("disk_free".equals(currentMetric.getMetricName())) {
- System.out.println("OUTPUT: " + currentMetric + " - " + currentHostAggregate);
- assertEquals(2, currentHostAggregate.getNumberOfHosts());
- assertEquals(5.0, Math.floor(currentHostAggregate.getSum()));
- recordCount++;
- } else {
- fail("Unexpected entry");
- }
- }
-
- Assert.assertEquals(5, recordCount);
- }
-
- @Test
- public void testShouldAggregateDifferentMetricsOnClusterProperly() throws Exception {
- // GIVEN
- TimelineMetricAggregator agg =
- TimelineMetricAggregatorFactory.createTimelineClusterAggregatorSecond(hdb,
- getConfigurationForTest(false), new TimelineMetricMetadataManager(new Configuration(), hdb), null, null);
- TimelineMetricReadHelper readHelper = new TimelineMetricReadHelper(false);
-
- // here we put some metrics tha will be aggregated
- long startTime = System.currentTimeMillis();
- long ctime = startTime;
- long minute = 60 * 1000;
- hdb.insertMetricRecords(prepareSingleTimelineMetric(ctime, "local1",
- "disk_free", 1));
- hdb.insertMetricRecords(prepareSingleTimelineMetric(ctime, "local2",
- "disk_free", 2));
- hdb.insertMetricRecords(prepareSingleTimelineMetric(ctime, "local1",
- "disk_used", 1));
-
- ctime += 2*minute;
- hdb.insertMetricRecords(prepareSingleTimelineMetric(ctime, "local1",
- "disk_free", 2));
- hdb.insertMetricRecords(prepareSingleTimelineMetric(ctime, "local2",
- "disk_free", 1));
- hdb.insertMetricRecords(prepareSingleTimelineMetric(ctime, "local1",
- "disk_used", 1));
-
- // WHEN
- long endTime = ctime + minute;
- boolean success = agg.doWork(startTime, endTime);
-
- //THEN
- Condition condition = new DefaultCondition(null, null, null, null, startTime,
- endTime, null, null, true);
- condition.setStatement(String.format(GET_CLUSTER_AGGREGATE_SQL,
- METRICS_CLUSTER_AGGREGATE_TABLE_NAME));
-
- PreparedStatement pstmt = PhoenixTransactSQL.prepareGetMetricsSqlStmt(conn, condition);
- ResultSet rs = pstmt.executeQuery();
-
- int recordCount = 0;
- while (rs.next()) {
- TimelineClusterMetric currentMetric = metricReader.fromResultSet(rs);
- MetricClusterAggregate currentHostAggregate =
- readHelper.getMetricClusterAggregateFromResultSet(rs);
-
- if ("disk_free".equals(currentMetric.getMetricName())) {
- assertEquals(2, currentHostAggregate.getNumberOfHosts());
- assertEquals(2.0, currentHostAggregate.getMax());
- assertEquals(1.0, currentHostAggregate.getMin());
- assertEquals(3.0, currentHostAggregate.getSum());
- recordCount++;
- } else if ("disk_used".equals(currentMetric.getMetricName())) {
- assertEquals(1, currentHostAggregate.getNumberOfHosts());
- assertEquals(1.0, currentHostAggregate.getMax());
- assertEquals(1.0, currentHostAggregate.getMin());
- assertEquals(1.0, currentHostAggregate.getSum());
- recordCount++;
- } else {
- fail("Unexpected entry");
- }
- }
- }
-
- @Test
- public void testAggregateDailyClusterMetrics() throws Exception {
- // GIVEN
- TimelineMetricAggregator agg =
- TimelineMetricAggregatorFactory.createTimelineClusterAggregatorDaily(hdb, getConfigurationForTest(false), null, null);
-
- // this time can be virtualized! or made independent from real clock
- long startTime = System.currentTimeMillis();
- long ctime = startTime;
- long hour = 3600 * 1000;
-
- Map<TimelineClusterMetric, MetricHostAggregate> records =
- new HashMap<TimelineClusterMetric, MetricHostAggregate>();
-
- records.put(createEmptyTimelineClusterMetric(ctime),
- MetricTestHelper.createMetricHostAggregate(4.0, 0.0, 2, 4.0));
- records.put(createEmptyTimelineClusterMetric(ctime += hour),
- MetricTestHelper.createMetricHostAggregate(4.0, 0.0, 2, 4.0));
- records.put(createEmptyTimelineClusterMetric(ctime += hour),
- MetricTestHelper.createMetricHostAggregate(4.0, 0.0, 2, 4.0));
- records.put(createEmptyTimelineClusterMetric(ctime += hour),
- MetricTestHelper.createMetricHostAggregate(4.0, 0.0, 2, 4.0));
-
-
- hdb.saveClusterAggregateRecordsSecond(records, METRICS_CLUSTER_AGGREGATE_HOURLY_TABLE_NAME);
-
- // WHEN
- agg.doWork(startTime, ctime + hour + 1000);
-
- // THEN
- ResultSet rs = executeQuery("SELECT * FROM METRIC_AGGREGATE_DAILY");
- int count = 0;
- while (rs.next()) {
- assertEquals("METRIC_NAME", "disk_used", rs.getString("METRIC_NAME"));
- assertEquals("APP_ID", "test_app", rs.getString("APP_ID"));
- assertEquals("METRIC_SUM", 16.0, rs.getDouble("METRIC_SUM"));
- assertEquals("METRIC_COUNT", 8, rs.getLong("METRIC_COUNT"));
- assertEquals("METRIC_MAX", 4.0, rs.getDouble("METRIC_MAX"));
- assertEquals("METRIC_MIN", 0.0, rs.getDouble("METRIC_MIN"));
- count++;
- }
-
- assertEquals("Day aggregated row expected ", 1, count);
- }
-
- @Test
- public void testShouldAggregateClusterOnMinuteProperly() throws Exception {
-
- TimelineMetricAggregator agg =
- TimelineMetricAggregatorFactory.createTimelineClusterAggregatorMinute(hdb, getConfigurationForTest(false), null, null);
-
- long startTime = System.currentTimeMillis();
- long ctime = startTime;
- long second = 1000;
- long minute = 60*second;
-
- Map<TimelineClusterMetric, MetricClusterAggregate> records =
- new HashMap<TimelineClusterMetric, MetricClusterAggregate>();
-
- records.put(createEmptyTimelineClusterMetric(ctime),
- new MetricClusterAggregate(4.0, 2, 0.0, 4.0, 0.0));
- records.put(createEmptyTimelineClusterMetric(ctime += second),
- new MetricClusterAggregate(4.0, 2, 0.0, 4.0, 0.0));
- records.put(createEmptyTimelineClusterMetric(ctime += second),
- new MetricClusterAggregate(4.0, 2, 0.0, 4.0, 0.0));
- records.put(createEmptyTimelineClusterMetric(ctime += second),
- new MetricClusterAggregate(4.0, 2, 0.0, 4.0, 0.0));
-
- hdb.saveClusterAggregateRecords(records);
- agg.doWork(startTime, ctime + second);
- long oldCtime = ctime + second;
-
- //Next minute
- ctime = startTime + minute;
-
- records.put(createEmptyTimelineClusterMetric(ctime),
- new MetricClusterAggregate(4.0, 2, 0.0, 4.0, 0.0));
- records.put(createEmptyTimelineClusterMetric(ctime += second),
- new MetricClusterAggregate(4.0, 2, 0.0, 4.0, 0.0));
- records.put(createEmptyTimelineClusterMetric(ctime += second),
- new MetricClusterAggregate(4.0, 2, 0.0, 4.0, 0.0));
- records.put(createEmptyTimelineClusterMetric(ctime += second),
- new MetricClusterAggregate(4.0, 2, 0.0, 4.0, 0.0));
-
- hdb.saveClusterAggregateRecords(records);
- agg.doWork(oldCtime, ctime + second);
-
- ResultSet rs = executeQuery("SELECT * FROM METRIC_AGGREGATE_MINUTE");
- int count = 0;
- long diff = 0 ;
- while (rs.next()) {
- assertEquals("METRIC_NAME", "disk_used", rs.getString("METRIC_NAME"));
- assertEquals("APP_ID", "test_app", rs.getString("APP_ID"));
- assertEquals("METRIC_SUM", 16.0, rs.getDouble("METRIC_SUM"));
- assertEquals("METRIC_COUNT", 8, rs.getLong("METRIC_COUNT"));
- assertEquals("METRIC_MAX", 4.0, rs.getDouble("METRIC_MAX"));
- assertEquals("METRIC_MIN", 0.0, rs.getDouble("METRIC_MIN"));
- if (count == 0) {
- diff+=rs.getLong("SERVER_TIME");
- } else {
- diff-=rs.getLong("SERVER_TIME");
- if (diff < 0) {
- diff*=-1;
- }
- assertTrue(diff == minute);
- }
- count++;
- }
-
- assertEquals("One hourly aggregated row expected ", 2, count);
- }
-
- @Test
- public void testShouldAggregateClusterOnHourProperly() throws Exception {
- // GIVEN
- TimelineMetricAggregator agg =
- TimelineMetricAggregatorFactory.createTimelineClusterAggregatorHourly(hdb, getConfigurationForTest(false), null, null);
-
- // this time can be virtualized! or made independent from real clock
- long startTime = System.currentTimeMillis();
- long ctime = startTime;
- long minute = 60 * 1000;
-
- Map<TimelineClusterMetric, MetricClusterAggregate> records =
- new HashMap<TimelineClusterMetric, MetricClusterAggregate>();
-
- records.put(createEmptyTimelineClusterMetric(ctime),
- new MetricClusterAggregate(4.0, 2, 0.0, 4.0, 0.0));
- records.put(createEmptyTimelineClusterMetric(ctime += minute),
- new MetricClusterAggregate(4.0, 2, 0.0, 4.0, 0.0));
- records.put(createEmptyTimelineClusterMetric(ctime += minute),
- new MetricClusterAggregate(4.0, 2, 0.0, 4.0, 0.0));
- records.put(createEmptyTimelineClusterMetric(ctime += minute),
- new MetricClusterAggregate(4.0, 2, 0.0, 4.0, 0.0));
-
- hdb.saveClusterAggregateRecords(records);
-
- // WHEN
- agg.doWork(startTime, ctime + minute);
-
- // THEN
- ResultSet rs = executeQuery("SELECT * FROM METRIC_AGGREGATE_HOURLY");
- int count = 0;
- while (rs.next()) {
- assertEquals("METRIC_NAME", "disk_used", rs.getString("METRIC_NAME"));
- assertEquals("APP_ID", "test_app", rs.getString("APP_ID"));
- assertEquals("METRIC_SUM", 16.0, rs.getDouble("METRIC_SUM"));
- assertEquals("METRIC_COUNT", 8, rs.getLong("METRIC_COUNT"));
- assertEquals("METRIC_MAX", 4.0, rs.getDouble("METRIC_MAX"));
- assertEquals("METRIC_MIN", 0.0, rs.getDouble("METRIC_MIN"));
- count++;
- }
-
- assertEquals("One hourly aggregated row expected ", 1, count);
- }
-
- @Test
- public void testShouldAggregateDifferentMetricsOnHourProperly() throws Exception {
- // GIVEN
- TimelineMetricAggregator agg =
- TimelineMetricAggregatorFactory.createTimelineClusterAggregatorHourly(hdb, getConfigurationForTest(false), null, null);
-
- long startTime = System.currentTimeMillis();
- long ctime = startTime;
- long minute = 60 * 1000;
-
- Map<TimelineClusterMetric, MetricClusterAggregate> records =
- new HashMap<TimelineClusterMetric, MetricClusterAggregate>();
-
- records.put(createEmptyTimelineClusterMetric("disk_used", ctime),
- new MetricClusterAggregate(4.0, 2, 0.0, 4.0, 0.0));
- records.put(createEmptyTimelineClusterMetric("disk_free", ctime),
- new MetricClusterAggregate(1.0, 2, 0.0, 1.0, 1.0));
-
- records.put(createEmptyTimelineClusterMetric("disk_used", ctime += minute),
- new MetricClusterAggregate(4.0, 2, 0.0, 4.0, 0.0));
- records.put(createEmptyTimelineClusterMetric("disk_free", ctime),
- new MetricClusterAggregate(1.0, 2, 0.0, 1.0, 1.0));
-
- records.put(createEmptyTimelineClusterMetric("disk_used", ctime += minute),
- new MetricClusterAggregate(4.0, 2, 0.0, 4.0, 0.0));
- records.put(createEmptyTimelineClusterMetric("disk_free", ctime),
- new MetricClusterAggregate(1.0, 2, 0.0, 1.0, 1.0));
-
- records.put(createEmptyTimelineClusterMetric("disk_used", ctime += minute),
- new MetricClusterAggregate(4.0, 2, 0.0, 4.0, 0.0));
- records.put(createEmptyTimelineClusterMetric("disk_free", ctime),
- new MetricClusterAggregate(1.0, 2, 0.0, 1.0, 1.0));
-
- hdb.saveClusterAggregateRecords(records);
-
- // WHEN
- agg.doWork(startTime, ctime + minute);
-
- // THEN
- ResultSet rs = executeQuery("SELECT * FROM METRIC_AGGREGATE_HOURLY");
- int count = 0;
- while (rs.next()) {
- if ("disk_used".equals(rs.getString("METRIC_NAME"))) {
- assertEquals("APP_ID", "test_app", rs.getString("APP_ID"));
- assertEquals("METRIC_SUM", 16.0, rs.getDouble("METRIC_SUM"));
- assertEquals("METRIC_COUNT", 8, rs.getLong("METRIC_COUNT"));
- assertEquals("METRIC_MAX", 4.0, rs.getDouble("METRIC_MAX"));
- assertEquals("METRIC_MIN", 0.0, rs.getDouble("METRIC_MIN"));
- } else if ("disk_free".equals(rs.getString("METRIC_NAME"))) {
- assertEquals("APP_ID", "test_app", rs.getString("APP_ID"));
- assertEquals("METRIC_SUM", 4.0, rs.getDouble("METRIC_SUM"));
- assertEquals("METRIC_COUNT", 8, rs.getLong("METRIC_COUNT"));
- assertEquals("METRIC_MAX", 1.0, rs.getDouble("METRIC_MAX"));
- assertEquals("METRIC_MIN", 1.0, rs.getDouble("METRIC_MIN"));
- }
-
- count++;
- }
-
- assertEquals("Two hourly aggregated row expected ", 2, count);
- }
-
- @Test
- public void testAppLevelHostMetricAggregates() throws Exception {
- Configuration conf = getConfigurationForTest(false);
- conf.set(CLUSTER_AGGREGATOR_APP_IDS, "app1");
- TimelineMetricAggregator agg =
- TimelineMetricAggregatorFactory.createTimelineClusterAggregatorSecond(hdb,
- conf, new TimelineMetricMetadataManager(new Configuration(), hdb), null, null);
- TimelineMetricReadHelper readHelper = new TimelineMetricReadHelper(false);
-
- long startTime = System.currentTimeMillis();
- long ctime = startTime;
- long minute = 60 * 1000;
- hdb.insertMetricRecords(prepareSingleTimelineMetric((ctime), "local1",
- "app1", null, "app_metric_random", 1));
- ctime += 10;
- hdb.insertMetricRecords(prepareSingleTimelineMetric(ctime, "local1",
- "cpu_user", 1));
- ctime += 10;
- hdb.insertMetricRecords(prepareSingleTimelineMetric(ctime, "local2",
- "cpu_user", 2));
-
- // WHEN
- long endTime = ctime + minute;
- boolean success = agg.doWork(startTime, endTime);
-
- //THEN
- Condition condition = new DefaultCondition(
- new ArrayList<String>() {{ add("cpu_user"); }}, null, "app1", null,
- startTime, endTime, null, null, true);
- condition.setStatement(String.format(GET_CLUSTER_AGGREGATE_SQL,
- METRICS_CLUSTER_AGGREGATE_TABLE_NAME));
-
- PreparedStatement pstmt = PhoenixTransactSQL.prepareGetMetricsSqlStmt(conn, condition);
- ResultSet rs = pstmt.executeQuery();
-
- int recordCount = 0;
- TimelineClusterMetric currentMetric = null;
- MetricClusterAggregate currentHostAggregate = null;
- while (rs.next()) {
- currentMetric = metricReader.fromResultSet(rs);
- currentHostAggregate = readHelper.getMetricClusterAggregateFromResultSet(rs);
- recordCount++;
- }
- assertEquals(3, recordCount);
- assertNotNull(currentMetric);
- assertEquals("cpu_user", currentMetric.getMetricName());
- assertEquals("app1", currentMetric.getAppId());
- assertNotNull(currentHostAggregate);
- assertEquals(1, currentHostAggregate.getNumberOfHosts());
- assertEquals(1.0d, currentHostAggregate.getSum());
- }
-
- @Test
- public void testClusterAggregateMetricNormalization() throws Exception {
- TimelineMetricAggregator agg =
- TimelineMetricAggregatorFactory.createTimelineClusterAggregatorSecond(hdb,
- getConfigurationForTest(false), new TimelineMetricMetadataManager(new Configuration(), hdb), null, null);
- TimelineMetricReadHelper readHelper = new TimelineMetricReadHelper(false);
-
- // Sample data
- TimelineMetric metric1 = new TimelineMetric();
- metric1.setMetricName("yarn.ClusterMetrics.NumActiveNMs");
- metric1.setAppId("resourcemanager");
- metric1.setHostName("h1");
- metric1.setStartTime(1431372311811l);
- metric1.setMetricValues(new TreeMap<Long, Double>() {{
- put(1431372311811l, 1.0);
- put(1431372321811l, 1.0);
- put(1431372331811l, 1.0);
- put(1431372341811l, 1.0);
- put(1431372351811l, 1.0);
- put(1431372361811l, 1.0);
- put(1431372371810l, 1.0);
- }});
-
- TimelineMetric metric2 = new TimelineMetric();
- metric2.setMetricName("yarn.ClusterMetrics.NumActiveNMs");
- metric2.setAppId("resourcemanager");
- metric2.setHostName("h1");
- metric2.setStartTime(1431372381810l);
- metric2.setMetricValues(new TreeMap<Long, Double>() {{
- put(1431372381810l, 1.0);
- put(1431372391811l, 1.0);
- put(1431372401811l, 1.0);
- put(1431372411811l, 1.0);
- put(1431372421811l, 1.0);
- put(1431372431811l, 1.0);
- put(1431372441810l, 1.0);
- }});
-
- TimelineMetrics metrics = new TimelineMetrics();
- metrics.setMetrics(Collections.singletonList(metric1));
- insertMetricRecords(conn, metrics, 1431372371810l);
-
- metrics.setMetrics(Collections.singletonList(metric2));
- insertMetricRecords(conn, metrics, 1431372441810l);
-
- long startTime = 1431372055000l;
- long endTime = 1431372655000l;
-
- agg.doWork(startTime, endTime);
-
- Condition condition = new DefaultCondition(null, null, null, null, startTime,
- endTime, null, null, true);
- condition.setStatement(String.format(GET_CLUSTER_AGGREGATE_SQL,
- METRICS_CLUSTER_AGGREGATE_TABLE_NAME));
-
- PreparedStatement pstmt = PhoenixTransactSQL.prepareGetMetricsSqlStmt(conn, condition);
- ResultSet rs = pstmt.executeQuery();
-
- int recordCount = 0;
- while (rs.next()) {
- TimelineClusterMetric currentMetric = metricReader.fromResultSet(rs);
- MetricClusterAggregate currentHostAggregate = readHelper.getMetricClusterAggregateFromResultSet(rs);
-
- if ("yarn.ClusterMetrics.NumActiveNMs".equals(currentMetric.getMetricName())) {
- assertEquals(1, currentHostAggregate.getNumberOfHosts());
- assertEquals(1.0, currentHostAggregate.getMax());
- assertEquals(1.0, currentHostAggregate.getMin());
- assertEquals(1.0, currentHostAggregate.getSum());
- recordCount++;
- } else {
- fail("Unexpected entry");
- }
- }
- Assert.assertEquals(5, recordCount);
- }
-
- @Test
- public void testAggregationUsingGroupByQuery() throws Exception {
- // GIVEN
- TimelineMetricAggregator agg =
- TimelineMetricAggregatorFactory.createTimelineClusterAggregatorHourly(hdb, getConfigurationForTest(true), null, null);
-
- long startTime = System.currentTimeMillis();
- long ctime = startTime;
- long minute = 60 * 1000;
-
- Map<TimelineClusterMetric, MetricClusterAggregate> records =
- new HashMap<TimelineClusterMetric, MetricClusterAggregate>();
-
- records.put(createEmptyTimelineClusterMetric("disk_used", ctime += minute),
- new MetricClusterAggregate(4.0, 2, 0.0, 4.0, 0.0));
- records.put(createEmptyTimelineClusterMetric("disk_free", ctime),
- new MetricClusterAggregate(1.0, 2, 0.0, 1.0, 1.0));
-
- records.put(createEmptyTimelineClusterMetric("disk_used", ctime += minute),
- new MetricClusterAggregate(4.0, 2, 0.0, 4.0, 0.0));
- records.put(createEmptyTimelineClusterMetric("disk_free", ctime),
- new MetricClusterAggregate(1.0, 2, 0.0, 1.0, 1.0));
-
- records.put(createEmptyTimelineClusterMetric("disk_used", ctime += minute),
- new MetricClusterAggregate(4.0, 2, 0.0, 4.0, 0.0));
- records.put(createEmptyTimelineClusterMetric("disk_free", ctime),
- new MetricClusterAggregate(1.0, 2, 0.0, 1.0, 1.0));
-
- records.put(createEmptyTimelineClusterMetric("disk_used", ctime += minute),
- new MetricClusterAggregate(4.0, 2, 0.0, 4.0, 0.0));
- records.put(createEmptyTimelineClusterMetric("disk_free", ctime),
- new MetricClusterAggregate(1.0, 2, 0.0, 1.0, 1.0));
-
- hdb.saveClusterAggregateRecords(records);
-
- // WHEN
- agg.doWork(startTime, ctime + minute);
-
- // THEN
- ResultSet rs = executeQuery("SELECT * FROM METRIC_AGGREGATE_HOURLY");
- int count = 0;
- while (rs.next()) {
- if ("disk_used".equals(rs.getString("METRIC_NAME"))) {
- assertEquals("APP_ID", "test_app", rs.getString("APP_ID"));
- assertEquals("METRIC_SUM", 4.0, rs.getDouble("METRIC_SUM"));
- assertEquals("METRIC_COUNT", 2, rs.getLong("METRIC_COUNT"));
- assertEquals("METRIC_MAX", 4.0, rs.getDouble("METRIC_MAX"));
- assertEquals("METRIC_MIN", 0.0, rs.getDouble("METRIC_MIN"));
- } else if ("disk_free".equals(rs.getString("METRIC_NAME"))) {
- assertEquals("APP_ID", "test_app", rs.getString("APP_ID"));
- assertEquals("METRIC_SUM", 1.0, rs.getDouble("METRIC_SUM"));
- assertEquals("METRIC_COUNT", 2, rs.getLong("METRIC_COUNT"));
- assertEquals("METRIC_MAX", 1.0, rs.getDouble("METRIC_MAX"));
- assertEquals("METRIC_MIN", 1.0, rs.getDouble("METRIC_MIN"));
- }
- count++;
- }
- assertEquals("Two hourly aggregated row expected ", 2, count);
- }
-
- private ResultSet executeQuery(String query) throws SQLException {
- Connection conn = getConnection(getUrl());
- Statement stmt = conn.createStatement();
- return stmt.executeQuery(query);
- }
-}
[21/28] ambari git commit: AMBARI-22740 : Fix integration test for
HBase in branch-3.0-ams due to UUID changes. (avijayan)
Posted by av...@apache.org.
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/query/PhoenixTransactSQL.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/query/PhoenixTransactSQL.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/query/PhoenixTransactSQL.java
new file mode 100644
index 0000000..4c4e673
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/query/PhoenixTransactSQL.java
@@ -0,0 +1,803 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.metrics.core.timeline.query;
+
+import java.sql.Connection;
+import java.sql.PreparedStatement;
+import java.sql.SQLException;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.ambari.metrics.core.timeline.discovery.TimelineMetricMetadataManager;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.metrics2.sink.timeline.Precision;
+import org.apache.hadoop.metrics2.sink.timeline.PrecisionLimitExceededException;
+import org.apache.ambari.metrics.core.timeline.PhoenixHBaseAccessor;
+
+/**
+ * Encapsulate all metrics related SQL queries.
+ */
+public class PhoenixTransactSQL {
+
+ public static final Log LOG = LogFactory.getLog(PhoenixTransactSQL.class);
+
+ /**
+ * Create table to store individual metric records.
+ */
+ public static final String CREATE_METRICS_TABLE_SQL = "CREATE TABLE IF NOT " +
+ "EXISTS METRIC_RECORD (UUID BINARY(20) NOT NULL, " +
+ "SERVER_TIME BIGINT NOT NULL, " +
+ "METRIC_SUM DOUBLE, " +
+ "METRIC_COUNT UNSIGNED_INT, " +
+ "METRIC_MAX DOUBLE, " +
+ "METRIC_MIN DOUBLE, " +
+ "METRICS VARCHAR CONSTRAINT pk " +
+ "PRIMARY KEY (UUID, SERVER_TIME ROW_TIMESTAMP)) DATA_BLOCK_ENCODING='%s', IMMUTABLE_ROWS=true, " +
+ "TTL=%s, COMPRESSION='%s'";
+
+ public static final String CREATE_CONTAINER_METRICS_TABLE_SQL =
+ "CREATE TABLE IF NOT EXISTS CONTAINER_METRICS "
+ + "(APP_ID VARCHAR, "
+ + " CONTAINER_ID VARCHAR,"
+ + " START_TIME TIMESTAMP,"
+ + " FINISH_TIME TIMESTAMP, "
+ + " DURATION BIGINT,"
+ + " HOSTNAME VARCHAR,"
+ + " EXIT_CODE INTEGER,"
+ + " LOCALIZATION_DURATION BIGINT,"
+ + " LAUNCH_DURATION BIGINT,"
+ + " MEM_REQUESTED_GB DOUBLE,"
+ + " MEM_REQUESTED_GB_MILLIS DOUBLE,"
+ + " MEM_VIRTUAL_GB DOUBLE,"
+ + " MEM_USED_GB_MIN DOUBLE,"
+ + " MEM_USED_GB_MAX DOUBLE,"
+ + " MEM_USED_GB_AVG DOUBLE,"
+ + " MEM_USED_GB_50_PCT DOUBLE,"
+ + " MEM_USED_GB_75_PCT DOUBLE,"
+ + " MEM_USED_GB_90_PCT DOUBLE,"
+ + " MEM_USED_GB_95_PCT DOUBLE,"
+ + " MEM_USED_GB_99_PCT DOUBLE,"
+ + " MEM_UNUSED_GB DOUBLE,"
+ + " MEM_UNUSED_GB_MILLIS DOUBLE "
+ + " CONSTRAINT pk PRIMARY KEY(APP_ID, CONTAINER_ID)) DATA_BLOCK_ENCODING='%s',"
+ + " IMMUTABLE_ROWS=true, TTL=%s, COMPRESSION='%s'";
+
+ public static final String CREATE_METRICS_AGGREGATE_TABLE_SQL =
+ "CREATE TABLE IF NOT EXISTS %s " +
+ "(UUID BINARY(20) NOT NULL, " +
+ "SERVER_TIME BIGINT NOT NULL, " +
+ "METRIC_SUM DOUBLE," +
+ "METRIC_COUNT UNSIGNED_INT, " +
+ "METRIC_MAX DOUBLE," +
+ "METRIC_MIN DOUBLE CONSTRAINT pk " +
+ "PRIMARY KEY (UUID, SERVER_TIME ROW_TIMESTAMP)) DATA_BLOCK_ENCODING='%s', IMMUTABLE_ROWS=true, TTL=%s," +
+ " COMPRESSION='%s'";
+
+ public static final String CREATE_METRICS_CLUSTER_AGGREGATE_TABLE_SQL =
+ "CREATE TABLE IF NOT EXISTS %s " +
+ "(UUID BINARY(16) NOT NULL, " +
+ "SERVER_TIME BIGINT NOT NULL, " +
+ "METRIC_SUM DOUBLE, " +
+ "HOSTS_COUNT UNSIGNED_INT, " +
+ "METRIC_MAX DOUBLE, " +
+ "METRIC_MIN DOUBLE " +
+ "CONSTRAINT pk PRIMARY KEY (UUID, SERVER_TIME ROW_TIMESTAMP)) DATA_BLOCK_ENCODING='%s', IMMUTABLE_ROWS=true, " +
+ "TTL=%s, COMPRESSION='%s'";
+
+ // HOSTS_COUNT vs METRIC_COUNT
+ public static final String CREATE_METRICS_CLUSTER_AGGREGATE_GROUPED_TABLE_SQL =
+ "CREATE TABLE IF NOT EXISTS %s " +
+ "(UUID BINARY(16) NOT NULL, " +
+ "SERVER_TIME BIGINT NOT NULL, " +
+ "METRIC_SUM DOUBLE, " +
+ "METRIC_COUNT UNSIGNED_INT, " +
+ "METRIC_MAX DOUBLE, " +
+ "METRIC_MIN DOUBLE " +
+ "CONSTRAINT pk PRIMARY KEY (UUID, SERVER_TIME ROW_TIMESTAMP)) DATA_BLOCK_ENCODING='%s', IMMUTABLE_ROWS=true, " +
+ "TTL=%s, COMPRESSION='%s'";
+
+ public static final String CREATE_METRICS_METADATA_TABLE_SQL =
+ "CREATE TABLE IF NOT EXISTS METRICS_METADATA " +
+ "(METRIC_NAME VARCHAR, " +
+ "APP_ID VARCHAR, " +
+ "INSTANCE_ID VARCHAR, " +
+ "UUID BINARY(16), " +
+ "UNITS CHAR(20), " +
+ "TYPE CHAR(20), " +
+ "START_TIME UNSIGNED_LONG, " +
+ "SUPPORTS_AGGREGATION BOOLEAN, " +
+ "IS_WHITELISTED BOOLEAN " +
+ "CONSTRAINT pk PRIMARY KEY (METRIC_NAME, APP_ID)) " +
+ "DATA_BLOCK_ENCODING='%s', COMPRESSION='%s'";
+
+ public static final String CREATE_HOSTED_APPS_METADATA_TABLE_SQL =
+ "CREATE TABLE IF NOT EXISTS HOSTED_APPS_METADATA " +
+ "(HOSTNAME VARCHAR, UUID BINARY(4), APP_IDS VARCHAR, " +
+ "CONSTRAINT pk PRIMARY KEY (HOSTNAME))" +
+ "DATA_BLOCK_ENCODING='%s', COMPRESSION='%s'";
+
+ public static final String CREATE_INSTANCE_HOST_TABLE_SQL =
+ "CREATE TABLE IF NOT EXISTS INSTANCE_HOST_METADATA " +
+ "(INSTANCE_ID VARCHAR, HOSTNAME VARCHAR, " +
+ "CONSTRAINT pk PRIMARY KEY (INSTANCE_ID, HOSTNAME))" +
+ "DATA_BLOCK_ENCODING='%s', COMPRESSION='%s'";
+
+ public static final String ALTER_METRICS_METADATA_TABLE =
+ "ALTER TABLE METRICS_METADATA ADD IF NOT EXISTS IS_WHITELISTED BOOLEAN";
+
+ /**
+ * ALTER table to set new options
+ */
+ public static final String ALTER_SQL = "ALTER TABLE %s SET TTL=%s";
+
+ /**
+ * Insert into metric records table.
+ */
+ public static final String UPSERT_METRICS_SQL = "UPSERT INTO %s " +
+ "(UUID, " +
+ "SERVER_TIME, " +
+ "METRIC_SUM, " +
+ "METRIC_MAX, " +
+ "METRIC_MIN, " +
+ "METRIC_COUNT, " +
+ "METRICS) VALUES " +
+ "(?, ?, ?, ?, ?, ?, ?)";
+
+ public static final String UPSERT_CONTAINER_METRICS_SQL = "UPSERT INTO %s " +
+ "(APP_ID,"
+ + " CONTAINER_ID,"
+ + " START_TIME,"
+ + " FINISH_TIME,"
+ + " DURATION,"
+ + " HOSTNAME,"
+ + " EXIT_CODE,"
+ + " LOCALIZATION_DURATION,"
+ + " LAUNCH_DURATION,"
+ + " MEM_REQUESTED_GB,"
+ + " MEM_REQUESTED_GB_MILLIS,"
+ + " MEM_VIRTUAL_GB,"
+ + " MEM_USED_GB_MIN,"
+ + " MEM_USED_GB_MAX,"
+ + " MEM_USED_GB_AVG,"
+ + " MEM_USED_GB_50_PCT,"
+ + " MEM_USED_GB_75_PCT,"
+ + " MEM_USED_GB_90_PCT,"
+ + " MEM_USED_GB_95_PCT,"
+ + " MEM_USED_GB_99_PCT,"
+ + " MEM_UNUSED_GB,"
+ + " MEM_UNUSED_GB_MILLIS) VALUES " +
+ "(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)";
+
+ public static final String UPSERT_CLUSTER_AGGREGATE_SQL = "UPSERT INTO " +
+ "%s (UUID, " +
+ "SERVER_TIME, " +
+ "METRIC_SUM, " +
+ "HOSTS_COUNT, " +
+ "METRIC_MAX, " +
+ "METRIC_MIN) " +
+ "VALUES (?, ?, ?, ?, ?, ?)";
+
+ public static final String UPSERT_CLUSTER_AGGREGATE_TIME_SQL = "UPSERT INTO" +
+ " %s (UUID, SERVER_TIME, " +
+ "METRIC_SUM, " +
+ "METRIC_COUNT, " +
+ "METRIC_MAX, " +
+ "METRIC_MIN) " +
+ "VALUES (?, ?, ?, ?, ?, ?)";
+
+ public static final String UPSERT_AGGREGATE_RECORD_SQL = "UPSERT INTO " +
+ "%s (UUID, " +
+ "SERVER_TIME, " +
+ "METRIC_SUM, " +
+ "METRIC_MAX, " +
+ "METRIC_MIN," +
+ "METRIC_COUNT) " +
+ "VALUES (?, ?, ?, ?, ?, ?)";
+
+ public static final String UPSERT_METADATA_SQL =
+ "UPSERT INTO METRICS_METADATA (METRIC_NAME, APP_ID, INSTANCE_ID, UUID, UNITS, TYPE, " +
+ "START_TIME, SUPPORTS_AGGREGATION, IS_WHITELISTED) " +
+ "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)";
+
+ public static final String UPSERT_HOSTED_APPS_METADATA_SQL =
+ "UPSERT INTO HOSTED_APPS_METADATA (HOSTNAME, UUID, APP_IDS) VALUES (?, ?, ?)";
+
+ public static final String UPSERT_INSTANCE_HOST_METADATA_SQL =
+ "UPSERT INTO INSTANCE_HOST_METADATA (INSTANCE_ID, HOSTNAME) VALUES (?, ?)";
+
+ /**
+ * Retrieve a set of rows from metrics records table.
+ */
+ public static final String GET_METRIC_SQL = "SELECT UUID, SERVER_TIME, " +
+ "METRIC_SUM, " +
+ "METRIC_MAX, " +
+ "METRIC_MIN, " +
+ "METRIC_COUNT, " +
+ "METRICS " +
+ "FROM %s";
+
+ /**
+ * Get latest metrics for a number of hosts
+ *
+ * Different queries for a number and a single hosts are used due to bug
+ * in Apache Phoenix
+ */
+ public static final String GET_LATEST_METRIC_SQL = "SELECT %s E.UUID AS UUID, " +
+ "E.SERVER_TIME AS SERVER_TIME, " +
+ "E.METRIC_SUM AS METRIC_SUM, " +
+ "E.METRIC_MAX AS METRIC_MAX, E.METRIC_MIN AS METRIC_MIN, " +
+ "E.METRIC_COUNT AS METRIC_COUNT, E.METRICS AS METRICS " +
+ "FROM %s AS E " +
+ "INNER JOIN " +
+ "(SELECT UUID, MAX(SERVER_TIME) AS MAX_SERVER_TIME " +
+ "FROM %s " +
+ "WHERE " +
+ "%s " +
+ "GROUP BY UUID) " +
+ "AS I " +
+ "ON E.UUID=I.UUID " +
+ "AND E.SERVER_TIME=I.MAX_SERVER_TIME";
+
+ public static final String GET_METRIC_AGGREGATE_ONLY_SQL = "SELECT UUID, " +
+ "SERVER_TIME, " +
+ "METRIC_SUM, " +
+ "METRIC_MAX, " +
+ "METRIC_MIN, " +
+ "METRIC_COUNT " +
+ "FROM %s";
+
+ public static final String GET_CLUSTER_AGGREGATE_SQL = "SELECT " +
+ "UUID, " +
+ "SERVER_TIME, " +
+ "METRIC_SUM, " +
+ "HOSTS_COUNT, " +
+ "METRIC_MAX, " +
+ "METRIC_MIN " +
+ "FROM %s";
+
+ public static final String GET_CLUSTER_AGGREGATE_TIME_SQL = "SELECT " +
+ "UUID, " +
+ "SERVER_TIME, " +
+ "METRIC_SUM, " +
+ "METRIC_COUNT, " +
+ "METRIC_MAX, " +
+ "METRIC_MIN " +
+ "FROM %s";
+
+ public static final String TOP_N_INNER_SQL = "SELECT UUID " +
+ "FROM %s WHERE %s GROUP BY UUID ORDER BY %s LIMIT %s";
+
+ public static final String GET_METRIC_METADATA_SQL = "SELECT " +
+ "METRIC_NAME, APP_ID, INSTANCE_ID, UUID, UNITS, TYPE, START_TIME, " +
+ "SUPPORTS_AGGREGATION, IS_WHITELISTED FROM METRICS_METADATA";
+
+ public static final String GET_HOSTED_APPS_METADATA_SQL = "SELECT " +
+ "HOSTNAME, UUID, APP_IDS FROM HOSTED_APPS_METADATA";
+
+ public static final String GET_INSTANCE_HOST_METADATA_SQL = "SELECT " +
+ "INSTANCE_ID, HOSTNAME FROM INSTANCE_HOST_METADATA";
+
+ /**
+ * Aggregate host metrics using a GROUP BY clause to take advantage of
+ * N - way parallel scan where N = number of regions.
+ */
+ public static final String GET_AGGREGATED_HOST_METRIC_GROUPBY_SQL = "UPSERT " +
+ "INTO %s (UUID, SERVER_TIME, METRIC_SUM, METRIC_COUNT, METRIC_MAX, METRIC_MIN) " +
+ "SELECT UUID, %s AS SERVER_TIME, " +
+ "SUM(METRIC_SUM), SUM(METRIC_COUNT), MAX(METRIC_MAX), MIN(METRIC_MIN) " +
+ "FROM %s WHERE%s SERVER_TIME > %s AND SERVER_TIME <= %s " +
+ "GROUP BY UUID";
+
+ /**
+ * Downsample host metrics.
+ */
+ public static final String DOWNSAMPLE_HOST_METRIC_SQL_UPSERT_PREFIX = "UPSERT INTO %s (UUID, SERVER_TIME, " +
+ "METRIC_SUM, METRIC_COUNT, METRIC_MAX, METRIC_MIN) ";
+
+ public static final String TOPN_DOWNSAMPLER_HOST_METRIC_SELECT_SQL = "SELECT UUID, " +
+ "%s AS SERVER_TIME, %s, 1, %s, %s FROM %s WHERE UUID IN %s AND SERVER_TIME > %s AND SERVER_TIME <= %s " +
+ "GROUP BY UUID ORDER BY %s DESC LIMIT %s";
+
+ /**
+ * Aggregate app metrics using a GROUP BY clause to take advantage of
+ * N - way parallel scan where N = number of regions.
+ */
+ public static final String GET_AGGREGATED_APP_METRIC_GROUPBY_SQL = "UPSERT " +
+ "INTO %s (UUID, SERVER_TIME, METRIC_SUM, METRIC_COUNT, METRIC_MAX, METRIC_MIN) SELECT UUID, %s AS SERVER_TIME, " +
+ "ROUND(AVG(METRIC_SUM),2), ROUND(AVG(%s)), MAX(METRIC_MAX), MIN(METRIC_MIN) FROM %s WHERE%s SERVER_TIME > %s AND " +
+ "SERVER_TIME <= %s GROUP BY UUID";
+
+ /**
+ * Downsample cluster metrics.
+ */
+ public static final String DOWNSAMPLE_CLUSTER_METRIC_SQL_UPSERT_PREFIX = "UPSERT INTO %s (UUID, SERVER_TIME, " +
+ "METRIC_SUM, METRIC_COUNT, METRIC_MAX, METRIC_MIN) ";
+
+ public static final String TOPN_DOWNSAMPLER_CLUSTER_METRIC_SELECT_SQL = "SELECT UUID, " +
+ "%s AS SERVER_TIME, %s, 1, %s, %s FROM %s WHERE UUID IN %s AND SERVER_TIME > %s AND SERVER_TIME <= %s " +
+ "GROUP BY UUID ORDER BY %s DESC LIMIT %s";
+
+ public static final String METRICS_RECORD_TABLE_NAME = "METRIC_RECORD";
+
+ public static final String CONTAINER_METRICS_TABLE_NAME = "CONTAINER_METRICS";
+
+ public static final String METRICS_AGGREGATE_MINUTE_TABLE_NAME =
+ "METRIC_RECORD_MINUTE";
+ public static final String METRICS_AGGREGATE_HOURLY_TABLE_NAME =
+ "METRIC_RECORD_HOURLY";
+ public static final String METRICS_AGGREGATE_DAILY_TABLE_NAME =
+ "METRIC_RECORD_DAILY";
+ public static final String METRICS_CLUSTER_AGGREGATE_TABLE_NAME =
+ "METRIC_AGGREGATE";
+ public static final String METRICS_CLUSTER_AGGREGATE_MINUTE_TABLE_NAME =
+ "METRIC_AGGREGATE_MINUTE";
+ public static final String METRICS_CLUSTER_AGGREGATE_HOURLY_TABLE_NAME =
+ "METRIC_AGGREGATE_HOURLY";
+ public static final String METRICS_CLUSTER_AGGREGATE_DAILY_TABLE_NAME =
+ "METRIC_AGGREGATE_DAILY";
+
+ public static final String PHOENIX_TABLES_REGEX_PATTERN_STRING = "METRIC_.*";
+
+ public static final String[] PHOENIX_TABLES = {
+ METRICS_RECORD_TABLE_NAME,
+ METRICS_AGGREGATE_MINUTE_TABLE_NAME,
+ METRICS_AGGREGATE_HOURLY_TABLE_NAME,
+ METRICS_AGGREGATE_DAILY_TABLE_NAME,
+ METRICS_CLUSTER_AGGREGATE_TABLE_NAME,
+ METRICS_CLUSTER_AGGREGATE_MINUTE_TABLE_NAME,
+ METRICS_CLUSTER_AGGREGATE_HOURLY_TABLE_NAME,
+ METRICS_CLUSTER_AGGREGATE_DAILY_TABLE_NAME
+ };
+
+ public static final String DEFAULT_TABLE_COMPRESSION = "SNAPPY";
+ public static final String DEFAULT_ENCODING = "FAST_DIFF";
+ public static final long HOUR = 3600000; // 1 hour
+ public static final long DAY = 86400000; // 1 day
+ private static boolean sortMergeJoinEnabled = false;
+
+ /**
+ * Filter to optimize HBase scan by using file timestamps. This prevents
+ * a full table scan of metric records.
+ *
+ * @return Phoenix Hint String
+ */
+ public static String getNaiveTimeRangeHint(Long startTime, Long delta) {
+ return String.format("/*+ NATIVE_TIME_RANGE(%s) */", (startTime - delta));
+ }
+
+ /**
+ * Falling back to sort merge join algorithm if default queries fail.
+ *
+ * @return Phoenix Hint String
+ */
+ public static String getLatestMetricsHints() {
+ if (sortMergeJoinEnabled) {
+ return "/*+ USE_SORT_MERGE_JOIN NO_CACHE */";
+ }
+ return "";
+ }
+
+ public static void setSortMergeJoinEnabled(boolean sortMergeJoinEnabled) {
+ PhoenixTransactSQL.sortMergeJoinEnabled = sortMergeJoinEnabled;
+ }
+
+ public static PreparedStatement prepareGetMetricsSqlStmt(Connection connection,
+ Condition condition) throws SQLException {
+
+ validateConditionIsNotEmpty(condition);
+ validateRowCountLimit(condition);
+
+ String stmtStr;
+ if (condition.getStatement() != null) {
+ stmtStr = condition.getStatement();
+ } else {
+ String metricsTable;
+ String query;
+ if (condition.getPrecision() == null) {
+ long endTime = condition.getEndTime() == null ? System.currentTimeMillis() : condition.getEndTime();
+ long startTime = condition.getStartTime() == null ? 0 : condition.getStartTime();
+ Precision precision = Precision.getPrecision(startTime, endTime);
+ condition.setPrecision(precision);
+ }
+ switch (condition.getPrecision()) {
+ case DAYS:
+ metricsTable = METRICS_AGGREGATE_DAILY_TABLE_NAME;
+ query = GET_METRIC_AGGREGATE_ONLY_SQL;
+ break;
+ case HOURS:
+ metricsTable = METRICS_AGGREGATE_HOURLY_TABLE_NAME;
+ query = GET_METRIC_AGGREGATE_ONLY_SQL;
+ break;
+ case MINUTES:
+ metricsTable = METRICS_AGGREGATE_MINUTE_TABLE_NAME;
+ query = GET_METRIC_AGGREGATE_ONLY_SQL;
+ break;
+ default:
+ metricsTable = METRICS_RECORD_TABLE_NAME;
+ query = GET_METRIC_SQL;
+ }
+
+ stmtStr = String.format(query, metricsTable);
+ }
+
+ StringBuilder sb = new StringBuilder(stmtStr);
+
+ if (!(condition instanceof EmptyCondition)) {
+ sb.append(" WHERE ");
+ sb.append(condition.getConditionClause());
+ String orderByClause = condition.getOrderByClause(true);
+ if (orderByClause != null) {
+ sb.append(orderByClause);
+ } else {
+ sb.append(" ORDER BY UUID, SERVER_TIME ");
+ }
+ }
+
+ if (condition.getLimit() != null) {
+ sb.append(" LIMIT ").append(condition.getLimit());
+ }
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("SQL: " + sb.toString() + ", condition: " + condition);
+ }
+
+ PreparedStatement stmt = null;
+ try {
+ stmt = connection.prepareStatement(sb.toString());
+ int pos = 1;
+ pos = addUuids(condition, pos, stmt);
+
+ if (condition instanceof TopNCondition) {
+ pos = addStartTime(condition, pos, stmt);
+ pos = addEndTime(condition, pos, stmt);
+ }
+
+ pos = addStartTime(condition, pos, stmt);
+ addEndTime(condition, pos, stmt);
+
+ if (condition.getFetchSize() != null) {
+ stmt.setFetchSize(condition.getFetchSize());
+ }
+ } catch (SQLException e) {
+ if (stmt != null) {
+ stmt.close();
+ }
+ throw e;
+ }
+
+ if (condition instanceof TopNCondition) {
+ LOG.info(sb.toString());
+ }
+ return stmt;
+ }
+
+ private static void validateConditionIsNotEmpty(Condition condition) {
+ if (condition.isEmpty()) {
+ throw new IllegalArgumentException("Condition is empty.");
+ }
+ }
+
+ private static void validateRowCountLimit(Condition condition) {
+ if (condition.getMetricNames() == null
+ || condition.getMetricNames().isEmpty()) {
+ //aggregator can use empty metrics query
+ return;
+ }
+
+ long range = condition.getEndTime() - condition.getStartTime();
+ long rowsPerMetric;
+
+ //Get Precision (passed in or computed) and estimate values returned based on that.
+ Precision precision = condition.getPrecision();
+ if (precision == null) {
+ precision = Precision.getPrecision(condition.getStartTime(), condition.getEndTime());
+ }
+
+ switch (precision) {
+ case DAYS:
+ rowsPerMetric = TimeUnit.MILLISECONDS.toDays(range);
+ break;
+ case HOURS:
+ rowsPerMetric = TimeUnit.MILLISECONDS.toHours(range);
+ break;
+ case MINUTES:
+ rowsPerMetric = TimeUnit.MILLISECONDS.toMinutes(range)/5; //5 minute data in METRIC_AGGREGATE_MINUTE table.
+ break;
+ default:
+ rowsPerMetric = TimeUnit.MILLISECONDS.toSeconds(range)/10; //10 second data in METRIC_AGGREGATE table
+ }
+
+ List<String> hostNames = condition.getHostnames();
+ int numHosts = (hostNames == null || hostNames.isEmpty()) ? 1 : condition.getHostnames().size();
+
+ long totalRowsRequested = rowsPerMetric * condition.getMetricNames().size() * numHosts;
+
+ if (totalRowsRequested > PhoenixHBaseAccessor.RESULTSET_LIMIT) {
+ throw new PrecisionLimitExceededException("Requested " + condition.getMetricNames().size() + " metrics for "
+ + numHosts + " hosts in " + precision + " precision for the time range of " + range/1000
+ + " seconds. Estimated resultset size of " + totalRowsRequested + " is greater than the limit of "
+ + PhoenixHBaseAccessor.RESULTSET_LIMIT + ". Request lower precision or fewer number of metrics or hosts." +
+ " Alternatively, increase the limit value through ams-site:timeline.metrics.service.default.result.limit config");
+ }
+ }
+
+ public static PreparedStatement prepareGetLatestMetricSqlStmt(
+ Connection connection, Condition condition) throws SQLException {
+
+ validateConditionIsNotEmpty(condition);
+
+ if (condition.getMetricNames() == null
+ || condition.getMetricNames().isEmpty()) {
+ throw new IllegalArgumentException("Point in time query without "
+ + "metric names not supported ");
+ }
+
+ String stmtStr;
+ if (condition.getStatement() != null) {
+ stmtStr = condition.getStatement();
+ } else {
+ stmtStr = String.format(GET_LATEST_METRIC_SQL,
+ getLatestMetricsHints(),
+ METRICS_RECORD_TABLE_NAME,
+ METRICS_RECORD_TABLE_NAME,
+ condition.getConditionClause());
+ }
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("SQL: " + stmtStr + ", condition: " + condition);
+ }
+ PreparedStatement stmt = null;
+ try {
+ stmt = connection.prepareStatement(stmtStr);
+ setQueryParameters(stmt, condition);
+ } catch (SQLException e) {
+ if (stmt != null) {
+ stmt.close();
+ }
+ throw e;
+ }
+
+ return stmt;
+ }
+
+ private static PreparedStatement setQueryParameters(PreparedStatement stmt,
+ Condition condition) throws SQLException {
+ int pos = 1;
+ //For GET_LATEST_METRIC_SQL_SINGLE_HOST parameters should be set 2 times
+ do {
+ if (condition.getUuids() != null) {
+ for (byte[] uuid : condition.getUuids()) {
+ stmt.setBytes(pos++, uuid);
+ }
+ }
+ if (condition.getFetchSize() != null) {
+ stmt.setFetchSize(condition.getFetchSize());
+ pos++;
+ }
+ } while (pos < stmt.getParameterMetaData().getParameterCount());
+
+ return stmt;
+ }
+
+ public static PreparedStatement prepareGetAggregateSqlStmt(
+ Connection connection, Condition condition) throws SQLException {
+
+ validateConditionIsNotEmpty(condition);
+ validateRowCountLimit(condition);
+
+ String metricsAggregateTable;
+ String queryStmt;
+ if (condition.getPrecision() == null) {
+ long endTime = condition.getEndTime() == null ? System.currentTimeMillis() : condition.getEndTime();
+ long startTime = condition.getStartTime() == null ? 0 : condition.getStartTime();
+ condition.setPrecision(Precision.getPrecision(startTime, endTime));
+ }
+ switch (condition.getPrecision()) {
+ case DAYS:
+ metricsAggregateTable = METRICS_CLUSTER_AGGREGATE_DAILY_TABLE_NAME;
+ queryStmt = GET_CLUSTER_AGGREGATE_TIME_SQL;
+ break;
+ case HOURS:
+ metricsAggregateTable = METRICS_CLUSTER_AGGREGATE_HOURLY_TABLE_NAME;
+ queryStmt = GET_CLUSTER_AGGREGATE_TIME_SQL;
+ break;
+ case MINUTES:
+ metricsAggregateTable = METRICS_CLUSTER_AGGREGATE_MINUTE_TABLE_NAME;
+ queryStmt = GET_CLUSTER_AGGREGATE_TIME_SQL;
+ break;
+ default:
+ metricsAggregateTable = METRICS_CLUSTER_AGGREGATE_TABLE_NAME;
+ queryStmt = GET_CLUSTER_AGGREGATE_SQL;
+ }
+
+ queryStmt = String.format(queryStmt, metricsAggregateTable);
+
+ StringBuilder sb = new StringBuilder(queryStmt);
+ sb.append(" WHERE ");
+ sb.append(condition.getConditionClause());
+ sb.append(" ORDER BY UUID, SERVER_TIME");
+ if (condition.getLimit() != null) {
+ sb.append(" LIMIT ").append(condition.getLimit());
+ }
+
+ String query = sb.toString();
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("SQL => " + query + ", condition => " + condition);
+ }
+ PreparedStatement stmt = null;
+ try {
+ stmt = connection.prepareStatement(query);
+ int pos = 1;
+
+ pos = addUuids(condition, pos, stmt);
+
+ if (condition instanceof TopNCondition) {
+ pos = addStartTime(condition, pos, stmt);
+ pos = addEndTime(condition, pos, stmt);
+ }
+
+ // TODO: Upper case all strings on POST
+ pos = addStartTime(condition, pos, stmt);
+ addEndTime(condition, pos, stmt);
+ } catch (SQLException e) {
+ if (stmt != null) {
+ stmt.close();
+ }
+ throw e;
+ }
+
+ if (condition instanceof TopNCondition) {
+ LOG.info(sb.toString());
+ }
+ return stmt;
+ }
+
+ public static PreparedStatement prepareGetLatestAggregateMetricSqlStmt(
+ Connection connection, SplitByMetricNamesCondition condition) throws SQLException {
+
+ validateConditionIsNotEmpty(condition);
+
+ String stmtStr;
+ if (condition.getStatement() != null) {
+ stmtStr = condition.getStatement();
+ } else {
+ stmtStr = String.format(GET_CLUSTER_AGGREGATE_SQL,
+ METRICS_CLUSTER_AGGREGATE_TABLE_NAME);
+ }
+
+ StringBuilder sb = new StringBuilder(stmtStr);
+ sb.append(" WHERE ");
+ sb.append(condition.getConditionClause());
+ String orderByClause = condition.getOrderByClause(false);
+ if (orderByClause != null) {
+ sb.append(orderByClause);
+ } else {
+ sb.append(" ORDER BY UUID DESC, SERVER_TIME DESC ");
+ }
+
+ sb.append(" LIMIT ").append(condition.getMetricNames().size());
+
+ String query = sb.toString();
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("SQL: " + query + ", condition: " + condition);
+ }
+
+ PreparedStatement stmt = null;
+ try {
+ stmt = connection.prepareStatement(query);
+ int pos = 1;
+ if (condition.getMetricNames() != null) {
+ for (; pos <= condition.getMetricNames().size(); pos++) {
+ stmt.setBytes(pos, condition.getCurrentUuid());
+ }
+ }
+ } catch (SQLException e) {
+ if (stmt != null) {
+
+ }
+ throw e;
+ }
+
+ return stmt;
+ }
+
+ public static String getTargetTableUsingPrecision(Precision precision, boolean withHosts) {
+
+ String inputTable = null;
+ if (precision != null) {
+ if (withHosts) {
+ switch (precision) {
+ case DAYS:
+ inputTable = PhoenixTransactSQL.METRICS_AGGREGATE_DAILY_TABLE_NAME;
+ break;
+ case HOURS:
+ inputTable = PhoenixTransactSQL.METRICS_AGGREGATE_HOURLY_TABLE_NAME;
+ break;
+ case MINUTES:
+ inputTable = PhoenixTransactSQL.METRICS_AGGREGATE_MINUTE_TABLE_NAME;
+ break;
+ default:
+ inputTable = PhoenixTransactSQL.METRICS_RECORD_TABLE_NAME;
+ }
+ } else {
+ switch (precision) {
+ case DAYS:
+ inputTable = PhoenixTransactSQL.METRICS_CLUSTER_AGGREGATE_DAILY_TABLE_NAME;
+ break;
+ case HOURS:
+ inputTable = PhoenixTransactSQL.METRICS_CLUSTER_AGGREGATE_HOURLY_TABLE_NAME;
+ break;
+ case MINUTES:
+ inputTable = PhoenixTransactSQL.METRICS_CLUSTER_AGGREGATE_MINUTE_TABLE_NAME;
+ break;
+ default:
+ inputTable = PhoenixTransactSQL.METRICS_CLUSTER_AGGREGATE_TABLE_NAME;
+ }
+ }
+ } else {
+ if (withHosts) {
+ inputTable = PhoenixTransactSQL.METRICS_RECORD_TABLE_NAME;
+ } else {
+ inputTable = PhoenixTransactSQL.METRICS_CLUSTER_AGGREGATE_TABLE_NAME;
+ }
+ }
+ return inputTable;
+ }
+
+ private static int addUuids(Condition condition, int pos, PreparedStatement stmt) throws SQLException {
+ if (condition.getUuids() != null) {
+ for (int pos2 = 1 ; pos2 <= condition.getUuids().size(); pos2++,pos++) {
+ byte[] uuid = condition.getUuids().get(pos2 - 1);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Setting pos: " + pos + ", value = " + new String(uuid));
+ }
+
+ if (uuid.length != TimelineMetricMetadataManager.HOSTNAME_UUID_LENGTH + TimelineMetricMetadataManager.TIMELINE_METRIC_UUID_LENGTH) {
+ stmt.setString(pos, new String(uuid));
+ } else {
+ stmt.setBytes(pos, uuid);
+ }
+ }
+ }
+ return pos;
+ }
+
+ private static int addStartTime(Condition condition, int pos, PreparedStatement stmt) throws SQLException {
+ if (condition.getStartTime() != null) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Setting pos: " + pos + ", value: " + condition.getStartTime());
+ }
+ stmt.setLong(pos++, condition.getStartTime());
+ }
+ return pos;
+ }
+
+ private static int addEndTime(Condition condition, int pos, PreparedStatement stmt) throws SQLException {
+
+ if (condition.getEndTime() != null) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Setting pos: " + pos + ", value: " + condition.getEndTime());
+ }
+ stmt.setLong(pos++, condition.getEndTime());
+ }
+ return pos;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/query/SplitByMetricNamesCondition.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/query/SplitByMetricNamesCondition.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/query/SplitByMetricNamesCondition.java
new file mode 100644
index 0000000..2f1697f
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/query/SplitByMetricNamesCondition.java
@@ -0,0 +1,189 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.metrics.core.timeline.query;
+
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.hadoop.metrics2.sink.timeline.Precision;
+// TODO get rid of this class
+public class SplitByMetricNamesCondition implements Condition {
+ private final Condition adaptee;
+ private byte[] currentUuid;
+ private boolean metricNamesNotCondition = false;
+
+ public SplitByMetricNamesCondition(Condition condition){
+ this.adaptee = condition;
+ }
+
+ @Override
+ public boolean isEmpty() {
+ return adaptee.isEmpty();
+ }
+
+ @Override
+ public List<byte[]> getUuids() {
+ return adaptee.getUuids();
+ }
+
+ @Override
+ public List<String> getMetricNames() {
+ return Collections.singletonList(new String(currentUuid));
+ }
+
+ @Override
+ public boolean isPointInTime() {
+ return adaptee.isPointInTime();
+ }
+
+ @Override
+ public boolean isGrouped() {
+ return adaptee.isGrouped();
+ }
+
+ @Override
+ public void setStatement(String statement) {
+ adaptee.setStatement(statement);
+ }
+
+ @Override
+ public List<String> getHostnames() {
+ return adaptee.getHostnames();
+ }
+
+ @Override
+ public Precision getPrecision() {
+ return adaptee.getPrecision();
+ }
+
+ @Override
+ public void setPrecision(Precision precision) {
+ adaptee.setPrecision(precision);
+ }
+
+ @Override
+ public String getAppId() {
+ return adaptee.getAppId();
+ }
+
+ @Override
+ public String getInstanceId() {
+ return adaptee.getInstanceId();
+ }
+
+ @Override
+ public StringBuilder getConditionClause() {
+ StringBuilder sb = new StringBuilder();
+ boolean appendConjunction = false;
+
+ if (getMetricNames() != null) {
+ for (String name : getMetricNames()) {
+ if (sb.length() > 1) {
+ sb.append(" OR ");
+ }
+ sb.append("UUID = ?");
+ }
+
+ appendConjunction = true;
+ }
+
+ appendConjunction = DefaultCondition.append(sb, appendConjunction,
+ getStartTime(), " SERVER_TIME >= ?");
+ DefaultCondition.append(sb, appendConjunction, getEndTime(),
+ " SERVER_TIME < ?");
+
+ return sb;
+ }
+
+ @Override
+ public String getOrderByClause(boolean asc) {
+ return adaptee.getOrderByClause(asc);
+ }
+
+ @Override
+ public String getStatement() {
+ return adaptee.getStatement();
+ }
+
+ @Override
+ public Long getStartTime() {
+ return adaptee.getStartTime();
+ }
+
+ @Override
+ public Long getEndTime() {
+ return adaptee.getEndTime();
+ }
+
+ @Override
+ public Integer getLimit() {
+ return adaptee.getLimit();
+ }
+
+ @Override
+ public Integer getFetchSize() {
+ return adaptee.getFetchSize();
+ }
+
+ @Override
+ public void setFetchSize(Integer fetchSize) {
+ adaptee.setFetchSize(fetchSize);
+ }
+
+ @Override
+ public void addOrderByColumn(String column) {
+ adaptee.addOrderByColumn(column);
+ }
+
+ @Override
+ public void setNoLimit() {
+ adaptee.setNoLimit();
+ }
+
+ @Override
+ public boolean doUpdate() {
+ return false;
+ }
+
+ public List<String> getOriginalMetricNames() {
+ return adaptee.getMetricNames();
+ }
+
+ public void setCurrentUuid(byte[] uuid) {
+ this.currentUuid = uuid;
+ }
+
+ public byte[] getCurrentUuid() {
+ return currentUuid;
+ }
+
+ @Override
+ public void setMetricNamesNotCondition(boolean metricNamesNotCondition) {
+ this.metricNamesNotCondition = metricNamesNotCondition;
+ }
+
+ @Override
+ public void setHostnamesNotCondition(boolean hostNamesNotCondition) {
+
+ }
+
+ @Override
+ public void setUuidNotCondition(boolean uuidNotCondition) {
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/query/TopNCondition.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/query/TopNCondition.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/query/TopNCondition.java
new file mode 100644
index 0000000..e951cd5
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/query/TopNCondition.java
@@ -0,0 +1,162 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.metrics.core.timeline.query;
+
+import java.util.List;
+
+import org.apache.ambari.metrics.core.timeline.aggregators.Function;
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.metrics2.sink.timeline.Precision;
+
+public class TopNCondition extends DefaultCondition{
+
+ private Integer topN;
+ private boolean isBottomN;
+ private Function topNFunction;
+ private static final Log LOG = LogFactory.getLog(TopNCondition.class);
+
+ public TopNCondition(List<byte[]> uuids, List<String> metricNames, List<String> hostnames, String appId,
+ String instanceId, Long startTime, Long endTime, Precision precision,
+ Integer limit, boolean grouped, Integer topN, Function topNFunction,
+ boolean isBottomN) {
+ super(uuids, metricNames, hostnames, appId, instanceId, startTime, endTime, precision, limit, grouped);
+ this.topN = topN;
+ this.isBottomN = isBottomN;
+ this.topNFunction = topNFunction;
+ }
+
+ @Override
+ public StringBuilder getConditionClause() {
+
+
+ if (!(isTopNHostCondition(metricNames, hostnames) || isTopNMetricCondition(metricNames, hostnames))) {
+ LOG.error("Unsupported TopN Operation requested. Query can have either multiple hosts or multiple metric names " +
+ "but not both.");
+ return null;
+ }
+
+ StringBuilder sb = new StringBuilder();
+ sb.append(" UUID IN (");
+ sb.append(getTopNInnerQuery());
+ sb.append(")");
+
+ boolean appendConjunction = true;
+ appendConjunction = append(sb, appendConjunction, getStartTime(), " SERVER_TIME >= ?");
+ append(sb, appendConjunction, getEndTime(), " SERVER_TIME < ?");
+
+ return sb;
+ }
+
+ public String getTopNInnerQuery() {
+ return String.format(PhoenixTransactSQL.TOP_N_INNER_SQL,
+ PhoenixTransactSQL.getTargetTableUsingPrecision(precision, CollectionUtils.isNotEmpty(hostnames)),
+ super.getConditionClause().toString(), getTopNOrderByClause(), topN);
+ }
+
+ private String getTopNOrderByClause() {
+
+ String orderByClause = getColumnSelect(this.topNFunction);
+ orderByClause += (isBottomN ? " ASC" : " DESC");
+ return orderByClause;
+ }
+
+ public static String getColumnSelect(Function topNFunction) {
+ String columnSelect = null;
+ if (topNFunction != null) {
+ switch (topNFunction.getReadFunction()) {
+ case AVG:
+ columnSelect = "ROUND(AVG(METRIC_SUM),2)";
+ break;
+ case SUM:
+ columnSelect = "SUM(METRIC_SUM)";
+ break;
+ default:
+ columnSelect = "MAX(METRIC_MAX)";
+ break;
+ }
+ }
+ if (columnSelect == null) {
+ columnSelect = "MAX(METRIC_MAX)";
+ }
+ return columnSelect;
+ }
+
+ public boolean isTopNHostCondition() {
+ return isTopNHostCondition(metricNames, hostnames);
+ }
+
+ public boolean isTopNMetricCondition() {
+ return isTopNMetricCondition(metricNames, hostnames);
+ }
+
+ /**
+ * Check if this is a case of Top N hosts condition
+ * @param metricNames A list of Strings.
+ * @param hostnames A list of Strings.
+ * @return True if it is a Case of Top N Hosts (1 Metric and H hosts).
+ */
+ public static boolean isTopNHostCondition(List<String> metricNames, List<String> hostnames) {
+ // Case 1 : 1 Metric, H hosts
+ // Select Top N or Bottom N host series based on 1 metric (max/avg/sum)
+ // Hostnames cannot be empty
+ // Only 1 metric allowed, without wildcards
+ return (CollectionUtils.isNotEmpty(hostnames) && metricNames.size() == 1 && !metricNamesHaveWildcard(metricNames));
+
+ }
+
+ /**
+ * Check if this is a case of Top N metrics condition
+ * @param metricNames A list of Strings.
+ * @param hostnames A list of Strings.
+ * @return True if it is a Case of Top N Metrics (M Metric and 1 or 0 host).
+ */
+ public static boolean isTopNMetricCondition(List<String> metricNames, List<String> hostnames) {
+ // Case 2 : M Metric names or Regex, 1 or No host
+ // Select Top N or Bottom N metric series based on metric values(max/avg/sum)
+ // MetricNames cannot be empty
+ // No host (aggregate) or 1 host allowed, without wildcards
+ return (CollectionUtils.isNotEmpty(metricNames) && (hostnames == null || hostnames.size() <= 1) &&
+ !hostNamesHaveWildcard(hostnames));
+ }
+
+ public Integer getTopN() {
+ return topN;
+ }
+
+ public void setTopN(Integer topN) {
+ this.topN = topN;
+ }
+
+ public boolean isBottomN() {
+ return isBottomN;
+ }
+
+ public void setIsBottomN(boolean isBottomN) {
+ this.isBottomN = isBottomN;
+ }
+
+ public Function getTopNFunction() {
+ return topNFunction;
+ }
+
+ public void setTopNFunction(Function topNFunction) {
+ this.topNFunction = topNFunction;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/sink/DefaultFSSinkProvider.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/sink/DefaultFSSinkProvider.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/sink/DefaultFSSinkProvider.java
new file mode 100644
index 0000000..52abc1e
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/sink/DefaultFSSinkProvider.java
@@ -0,0 +1,151 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.metrics.core.timeline.sink;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.Collection;
+import java.util.Date;
+
+import org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguration;
+import org.apache.ambari.metrics.core.timeline.source.InternalSourceProvider;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.lang.StringUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.metrics2.sink.timeline.TimelineMetric;
+import org.apache.hadoop.metrics2.sink.timeline.TimelineMetrics;
+
+public class DefaultFSSinkProvider implements ExternalSinkProvider {
+ private static final Log LOG = LogFactory.getLog(DefaultFSSinkProvider.class);
+ TimelineMetricConfiguration conf = TimelineMetricConfiguration.getInstance();
+ private final DefaultExternalMetricsSink sink = new DefaultExternalMetricsSink();
+ private long FIXED_FILE_SIZE;
+ private final String SINK_FILE_NAME = "external-metrics-sink.dat";
+ private final String SEPARATOR = ", ";
+ private final String LINE_SEP = System.lineSeparator();
+ private final String HEADERS = "METRIC, APP_ID, INSTANCE_ID, HOSTNAME, START_TIME, DATA";
+
+ public DefaultFSSinkProvider() {
+ try {
+ FIXED_FILE_SIZE = conf.getMetricsConf().getLong("timeline.metrics.service.external.fs.sink.filesize", FileUtils.ONE_MB * 100);
+ } catch (Exception ignored) {
+ FIXED_FILE_SIZE = FileUtils.ONE_MB * 100;
+ }
+ }
+
+ @Override
+ public ExternalMetricsSink getExternalMetricsSink(InternalSourceProvider.SOURCE_NAME sourceName) {
+ return sink;
+ }
+
+ class DefaultExternalMetricsSink implements ExternalMetricsSink {
+
+ @Override
+ public int getSinkTimeOutSeconds() {
+ return 10;
+ }
+
+ @Override
+ public int getFlushSeconds() {
+ try {
+ return conf.getMetricsConf().getInt(TimelineMetricConfiguration.TIMELINE_METRICS_CACHE_COMMIT_INTERVAL, 3);
+ } catch (Exception e) {
+ LOG.warn("Cannot read cache commit interval.");
+ }
+ return 3;
+ }
+
+ private boolean createFile(File f) {
+ boolean created = false;
+ if (!f.exists()) {
+ try {
+ created = f.createNewFile();
+ FileUtils.writeStringToFile(f, HEADERS);
+ } catch (IOException e) {
+ LOG.error("Cannot create " + SINK_FILE_NAME + " at " + f.getPath());
+ return false;
+ }
+ }
+
+ return created;
+ }
+
+ private boolean shouldReCreate(File f) {
+ if (!f.exists()) {
+ return true;
+ }
+ if (FileUtils.sizeOf(f) > FIXED_FILE_SIZE) {
+ return true;
+ }
+ return false;
+ }
+
+ @Override
+ public void sinkMetricData(Collection<TimelineMetrics> metrics) {
+ String dirPath = TimelineMetricConfiguration.getInstance().getDefaultMetricsSinkDir();
+ File dir = new File(dirPath);
+ if (!dir.exists()) {
+ LOG.error("Cannot sink data to file system, incorrect dir path " + dirPath);
+ return;
+ }
+
+ File f = FileUtils.getFile(dirPath, SINK_FILE_NAME);
+ if (shouldReCreate(f)) {
+ if (!f.delete()) {
+ LOG.warn("Unable to delete external sink file.");
+ return;
+ }
+ createFile(f);
+ }
+
+ if (metrics != null) {
+ for (TimelineMetrics timelineMetrics : metrics) {
+ for (TimelineMetric metric : timelineMetrics.getMetrics()) {
+ StringBuilder sb = new StringBuilder();
+ sb.append(metric.getMetricName());
+ sb.append(SEPARATOR);
+ sb.append(metric.getAppId());
+ sb.append(SEPARATOR);
+ if (StringUtils.isEmpty(metric.getInstanceId())) {
+ sb.append(SEPARATOR);
+ } else {
+ sb.append(metric.getInstanceId());
+ sb.append(SEPARATOR);
+ }
+ if (StringUtils.isEmpty(metric.getHostName())) {
+ sb.append(SEPARATOR);
+ } else {
+ sb.append(metric.getHostName());
+ sb.append(SEPARATOR);
+ }
+ sb.append(new Date(metric.getStartTime()));
+ sb.append(SEPARATOR);
+ sb.append(metric.getMetricValues().toString());
+ sb.append(LINE_SEP);
+ try {
+ FileUtils.writeStringToFile(f, sb.toString());
+ } catch (IOException e) {
+ LOG.warn("Unable to sink data to file " + f.getPath());
+ }
+ }
+ }
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/sink/ExternalMetricsSink.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/sink/ExternalMetricsSink.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/sink/ExternalMetricsSink.java
new file mode 100644
index 0000000..b2bb908
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/sink/ExternalMetricsSink.java
@@ -0,0 +1,48 @@
+package org.apache.ambari.metrics.core.timeline.sink;
+
+import java.util.Collection;
+
+import org.apache.hadoop.metrics2.sink.timeline.TimelineMetrics;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+public interface ExternalMetricsSink {
+ /**
+ * How many seconds to wait on sink before dropping metrics.
+ * Note: Care should be taken that this timeout does not bottleneck the
+ * sink thread.
+ */
+ int getSinkTimeOutSeconds();
+
+ /**
+ * How frequently to flush data to external system.
+ * Default would be between 60 - 120 seconds, coherent with default sink
+ * interval of AMS.
+ */
+ int getFlushSeconds();
+
+ /**
+ * Raw data stream to process / store on external system.
+ * The data will be held in an in-memory cache and flushed at flush seconds
+ * or when the cache size limit is exceeded we will flush the cache and
+ * drop data if write fails.
+ *
+ * @param metrics {@link Collection<TimelineMetrics>}
+ */
+ void sinkMetricData(Collection<TimelineMetrics> metrics);
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/sink/ExternalSinkProvider.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/sink/ExternalSinkProvider.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/sink/ExternalSinkProvider.java
new file mode 100644
index 0000000..83cdc33
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/sink/ExternalSinkProvider.java
@@ -0,0 +1,35 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.metrics.core.timeline.sink;
+
+import org.apache.ambari.metrics.core.timeline.source.InternalSourceProvider;
+
+
+/**
+ * Configurable provider for sink classes that match the metrics sources.
+ * Provider can return same sink or different sinks for each source.
+ */
+public interface ExternalSinkProvider {
+
+ /**
+ * Return an instance of the metrics sink for the give source
+ * @return {@link ExternalMetricsSink}
+ */
+ ExternalMetricsSink getExternalMetricsSink(InternalSourceProvider.SOURCE_NAME sourceName);
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/sink/HttpSinkProvider.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/sink/HttpSinkProvider.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/sink/HttpSinkProvider.java
new file mode 100644
index 0000000..cbb0834
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/sink/HttpSinkProvider.java
@@ -0,0 +1,231 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.metrics.core.timeline.sink;
+
+import static org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguration.TIMELINE_METRICS_CACHE_COMMIT_INTERVAL;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.OutputStream;
+import java.net.HttpURLConnection;
+import java.net.URL;
+import java.security.KeyStore;
+import java.util.Collection;
+
+import javax.net.ssl.HttpsURLConnection;
+import javax.net.ssl.SSLContext;
+import javax.net.ssl.SSLSocketFactory;
+import javax.net.ssl.TrustManagerFactory;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.metrics2.sink.timeline.TimelineMetrics;
+import org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguration;
+import org.apache.ambari.metrics.core.timeline.source.InternalSourceProvider;
+import org.apache.http.client.utils.URIBuilder;
+import org.codehaus.jackson.map.AnnotationIntrospector;
+import org.codehaus.jackson.map.ObjectMapper;
+import org.codehaus.jackson.map.annotate.JsonSerialize;
+import org.codehaus.jackson.xc.JaxbAnnotationIntrospector;
+
+public class HttpSinkProvider implements ExternalSinkProvider {
+ private static final Log LOG = LogFactory.getLog(HttpSinkProvider.class);
+ TimelineMetricConfiguration conf = TimelineMetricConfiguration.getInstance();
+
+ private String connectUrl;
+ private SSLSocketFactory sslSocketFactory;
+ protected static ObjectMapper mapper;
+
+ static {
+ mapper = new ObjectMapper();
+ AnnotationIntrospector introspector = new JaxbAnnotationIntrospector();
+ mapper.setAnnotationIntrospector(introspector);
+ mapper.getSerializationConfig()
+ .withSerializationInclusion(JsonSerialize.Inclusion.NON_NULL);
+ }
+
+ public HttpSinkProvider() {
+ Configuration config;
+ try {
+ config = conf.getMetricsConf();
+ } catch (Exception e) {
+ throw new ExceptionInInitializerError("Unable to read configuration for sink.");
+ }
+ String protocol = config.get("timeline.metrics.service.external.http.sink.protocol", "http");
+ String host = config.get("timeline.metrics.service.external.http.sink.host", "localhost");
+ String port = config.get("timeline.metrics.service.external.http.sink.port", "6189");
+
+ if (protocol.contains("https")) {
+ loadTruststore(
+ config.getTrimmed("timeline.metrics.service.external.http.sink.truststore.path"),
+ config.getTrimmed("timeline.metrics.service.external.http.sink.truststore.type"),
+ config.getTrimmed("timeline.metrics.service.external.http.sink.truststore.password")
+ );
+ }
+
+ URIBuilder uriBuilder = new URIBuilder();
+ uriBuilder.setScheme(protocol);
+ uriBuilder.setHost(host);
+ uriBuilder.setPort(Integer.parseInt(port));
+ connectUrl = uriBuilder.toString();
+ }
+
+ @Override
+ public ExternalMetricsSink getExternalMetricsSink(InternalSourceProvider.SOURCE_NAME sourceName) {
+ return new DefaultHttpMetricsSink();
+ }
+
+ protected HttpURLConnection getConnection(String spec) throws IOException {
+ return (HttpURLConnection) new URL(spec).openConnection();
+ }
+
+ // Get an ssl connection
+ protected HttpsURLConnection getSSLConnection(String spec)
+ throws IOException, IllegalStateException {
+
+ HttpsURLConnection connection = (HttpsURLConnection) (new URL(spec).openConnection());
+ connection.setSSLSocketFactory(sslSocketFactory);
+ return connection;
+ }
+
+ protected void loadTruststore(String trustStorePath, String trustStoreType,
+ String trustStorePassword) {
+ if (sslSocketFactory == null) {
+ if (trustStorePath == null || trustStorePassword == null) {
+ String msg = "Can't load TrustStore. Truststore path or password is not set.";
+ LOG.error(msg);
+ throw new IllegalStateException(msg);
+ }
+ FileInputStream in = null;
+ try {
+ in = new FileInputStream(new File(trustStorePath));
+ KeyStore store = KeyStore.getInstance(trustStoreType == null ?
+ KeyStore.getDefaultType() : trustStoreType);
+ store.load(in, trustStorePassword.toCharArray());
+ TrustManagerFactory tmf = TrustManagerFactory
+ .getInstance(TrustManagerFactory.getDefaultAlgorithm());
+ tmf.init(store);
+ SSLContext context = SSLContext.getInstance("TLS");
+ context.init(null, tmf.getTrustManagers(), null);
+ sslSocketFactory = context.getSocketFactory();
+ } catch (Exception e) {
+ LOG.error("Unable to load TrustStore", e);
+ } finally {
+ if (in != null) {
+ try {
+ in.close();
+ } catch (IOException e) {
+ LOG.error("Unable to load TrustStore", e);
+ }
+ }
+ }
+ }
+ }
+
+ class DefaultHttpMetricsSink implements ExternalMetricsSink {
+
+ @Override
+ public int getSinkTimeOutSeconds() {
+ try {
+ return conf.getMetricsConf().getInt("timeline.metrics.external.sink.http.timeout.seconds", 10);
+ } catch (Exception e) {
+ return 10;
+ }
+ }
+
+ @Override
+ public int getFlushSeconds() {
+ try {
+ return conf.getMetricsConf().getInt(TIMELINE_METRICS_CACHE_COMMIT_INTERVAL, 3);
+ } catch (Exception e) {
+ LOG.warn("Cannot read cache commit interval.");
+ }
+ return 3;
+ }
+
+ /**
+ * Cleans up and closes an input stream
+ * see http://docs.oracle.com/javase/6/docs/technotes/guides/net/http-keepalive.html
+ * @param is the InputStream to clean up
+ * @return string read from the InputStream
+ * @throws IOException
+ */
+ protected String cleanupInputStream(InputStream is) throws IOException {
+ StringBuilder sb = new StringBuilder();
+ if (is != null) {
+ try (
+ InputStreamReader isr = new InputStreamReader(is);
+ BufferedReader br = new BufferedReader(isr)
+ ) {
+ // read the response body
+ String line;
+ while ((line = br.readLine()) != null) {
+ if (LOG.isDebugEnabled()) {
+ sb.append(line);
+ }
+ }
+ } finally {
+ is.close();
+ }
+ }
+ return sb.toString();
+ }
+
+ @Override
+ public void sinkMetricData(Collection<TimelineMetrics> metrics) {
+ HttpURLConnection connection = null;
+ try {
+ connection = connectUrl.startsWith("https") ? getSSLConnection(connectUrl) : getConnection(connectUrl);
+
+ connection.setRequestMethod("POST");
+ connection.setRequestProperty("Content-Type", "application/json");
+ connection.setRequestProperty("Connection", "Keep-Alive");
+ connection.setConnectTimeout(getSinkTimeOutSeconds());
+ connection.setReadTimeout(getSinkTimeOutSeconds());
+ connection.setDoOutput(true);
+
+ if (metrics != null) {
+ String jsonData = mapper.writeValueAsString(metrics);
+ try (OutputStream os = connection.getOutputStream()) {
+ os.write(jsonData.getBytes("UTF-8"));
+ }
+ }
+
+ int statusCode = connection.getResponseCode();
+
+ if (statusCode != 200) {
+ LOG.info("Unable to POST metrics to external sink, " + connectUrl +
+ ", statusCode = " + statusCode);
+ } else {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Metrics posted to external sink " + connectUrl);
+ }
+ }
+ cleanupInputStream(connection.getInputStream());
+
+ } catch (IOException io) {
+ LOG.warn("Unable to sink data to external system.", io);
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/sink/KafkaSinkProvider.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/sink/KafkaSinkProvider.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/sink/KafkaSinkProvider.java
new file mode 100644
index 0000000..1ce624b
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/sink/KafkaSinkProvider.java
@@ -0,0 +1,109 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.metrics.core.timeline.sink;
+
+import java.util.Collection;
+import java.util.Properties;
+import java.util.concurrent.Future;
+
+import org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguration;
+import org.apache.ambari.metrics.core.timeline.source.InternalSourceProvider;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.metrics2.sink.timeline.TimelineMetrics;
+import org.apache.kafka.clients.producer.KafkaProducer;
+import org.apache.kafka.clients.producer.Producer;
+import org.apache.kafka.clients.producer.ProducerConfig;
+import org.apache.kafka.clients.producer.ProducerRecord;
+import org.apache.kafka.clients.producer.RecordMetadata;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+
+/*
+ This will be used by the single Metrics committer thread. Hence it is
+ important to make this non-blocking export.
+ */
+public class KafkaSinkProvider implements ExternalSinkProvider {
+ private static String TOPIC_NAME = "ambari-metrics-topic";
+ private static final Log LOG = LogFactory.getLog(KafkaSinkProvider.class);
+
+ private Producer producer;
+ private int TIMEOUT_SECONDS = 10;
+ private int FLUSH_SECONDS = 3;
+
+ ObjectMapper objectMapper = new ObjectMapper();
+
+ public KafkaSinkProvider() {
+ TimelineMetricConfiguration configuration = TimelineMetricConfiguration.getInstance();
+
+ Properties configProperties = new Properties();
+ try {
+ configProperties.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, configuration.getMetricsConf().getTrimmed(TimelineMetricConfiguration.KAFKA_SERVERS));
+ configProperties.put(ProducerConfig.ACKS_CONFIG, configuration.getMetricsConf().getTrimmed(TimelineMetricConfiguration.KAFKA_ACKS, "all"));
+ // Avoid duplicates - No transactional semantics
+ configProperties.put(ProducerConfig.RETRIES_CONFIG, configuration.getMetricsConf().getInt(TimelineMetricConfiguration.KAFKA_RETRIES, 0));
+ configProperties.put(ProducerConfig.BATCH_SIZE_CONFIG, configuration.getMetricsConf().getInt(TimelineMetricConfiguration.KAFKA_BATCH_SIZE, 128));
+ configProperties.put(ProducerConfig.LINGER_MS_CONFIG, configuration.getMetricsConf().getInt(TimelineMetricConfiguration.KAFKA_LINGER_MS, 1));
+ configProperties.put(ProducerConfig.BUFFER_MEMORY_CONFIG, configuration.getMetricsConf().getLong(TimelineMetricConfiguration.KAFKA_BUFFER_MEM, 33554432)); // 32 MB
+ FLUSH_SECONDS = configuration.getMetricsConf().getInt(TimelineMetricConfiguration.TIMELINE_METRICS_CACHE_COMMIT_INTERVAL, 3);
+ TIMEOUT_SECONDS = configuration.getMetricsConf().getInt(TimelineMetricConfiguration.KAFKA_SINK_TIMEOUT_SECONDS, 10);
+ } catch (Exception e) {
+ LOG.error("Configuration error!", e);
+ throw new ExceptionInInitializerError(e);
+ }
+ configProperties.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG,"org.apache.kafka.common.serialization.ByteArraySerializer");
+ configProperties.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG,"org.apache.kafka.connect.json.JsonSerializer");
+
+
+
+ producer = new KafkaProducer(configProperties);
+ }
+
+ @Override
+ public ExternalMetricsSink getExternalMetricsSink(InternalSourceProvider.SOURCE_NAME sourceName) {
+ switch (sourceName) {
+ case RAW_METRICS:
+ return new KafkaRawMetricsSink();
+ default:
+ throw new UnsupportedOperationException("Provider does not support " +
+ "the expected source " + sourceName);
+ }
+ }
+
+ class KafkaRawMetricsSink implements ExternalMetricsSink {
+
+ @Override
+ public int getSinkTimeOutSeconds() {
+ return TIMEOUT_SECONDS;
+ }
+
+ @Override
+ public int getFlushSeconds() {
+ return FLUSH_SECONDS;
+ }
+
+ @Override
+ public void sinkMetricData(Collection<TimelineMetrics> metrics) {
+ JsonNode jsonNode = objectMapper.valueToTree(metrics);
+ ProducerRecord<String, JsonNode> rec = new ProducerRecord<String, JsonNode>(TOPIC_NAME, jsonNode);
+ Future<RecordMetadata> f = producer.send(rec);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/source/DefaultInternalMetricsSourceProvider.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/source/DefaultInternalMetricsSourceProvider.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/source/DefaultInternalMetricsSourceProvider.java
new file mode 100644
index 0000000..1287328
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/source/DefaultInternalMetricsSourceProvider.java
@@ -0,0 +1,42 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.metrics.core.timeline.source;
+
+import org.apache.ambari.metrics.core.timeline.sink.ExternalMetricsSink;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+public class DefaultInternalMetricsSourceProvider implements InternalSourceProvider {
+ private static final Log LOG = LogFactory.getLog(DefaultInternalMetricsSourceProvider.class);
+
+ // TODO: Implement read based sources for higher order data
+ @Override
+ public InternalMetricsSource getInternalMetricsSource(SOURCE_NAME sourceName, int sinkIntervalSeconds, ExternalMetricsSink sink) {
+ if (sink == null) {
+ LOG.warn("No external sink configured for source " + sourceName);
+ return null;
+ }
+
+ switch (sourceName) {
+ case RAW_METRICS:
+ return new RawMetricsSource(sinkIntervalSeconds, sink);
+ default:
+ throw new UnsupportedOperationException("Unimplemented source type " + sourceName);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/source/InternalMetricsSource.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/source/InternalMetricsSource.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/source/InternalMetricsSource.java
new file mode 100644
index 0000000..5aaf128
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/source/InternalMetricsSource.java
@@ -0,0 +1,30 @@
+package org.apache.ambari.metrics.core.timeline.source;
+
+import java.util.Collection;
+
+import org.apache.hadoop.metrics2.sink.timeline.TimelineMetrics;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+public interface InternalMetricsSource {
+ /**
+ * Write metrics to external sink.
+ * Allows pre-processing and caching capabilities to the consumer.
+ */
+ void publishTimelineMetrics(Collection<TimelineMetrics> metrics);
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/source/InternalSourceProvider.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/source/InternalSourceProvider.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/source/InternalSourceProvider.java
new file mode 100644
index 0000000..3bd8e42
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/source/InternalSourceProvider.java
@@ -0,0 +1,39 @@
+package org.apache.ambari.metrics.core.timeline.source;
+
+import org.apache.ambari.metrics.core.timeline.sink.ExternalMetricsSink;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+public interface InternalSourceProvider {
+
+ enum SOURCE_NAME {
+ RAW_METRICS,
+ MINUTE_HOST_AGGREAGATE_METRICS,
+ HOURLY_HOST_AGGREAGATE_METRICS,
+ DAILY_HOST_AGGREAGATE_METRICS,
+ MINUTE_CLUSTER_AGGREAGATE_METRICS,
+ HOURLY_CLUSTER_AGGREAGATE_METRICS,
+ DAILY_CLUSTER_AGGREAGATE_METRICS,
+ }
+
+ /**
+ * Provide Source for metrics data.
+ * @return {@link InternalMetricsSource}
+ */
+ InternalMetricsSource getInternalMetricsSource(SOURCE_NAME sourceName, int sinkIntervalSeconds, ExternalMetricsSink sink);
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/source/RawMetricsSource.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/source/RawMetricsSource.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/source/RawMetricsSource.java
new file mode 100644
index 0000000..9dddb33
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/source/RawMetricsSource.java
@@ -0,0 +1,85 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.metrics.core.timeline.source;
+
+import java.util.Collection;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+
+import org.apache.ambari.metrics.core.timeline.sink.ExternalMetricsSink;
+import org.apache.ambari.metrics.core.timeline.source.cache.InternalMetricsCacheProvider;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.metrics2.sink.timeline.TimelineMetrics;
+import org.apache.ambari.metrics.core.timeline.source.cache.InternalMetricsCache;
+
+public class RawMetricsSource implements InternalMetricsSource {
+ private static final Log LOG = LogFactory.getLog(RawMetricsSource.class);
+ private final int internalCacheInterval;
+ private final ExternalMetricsSink rawMetricsSink;
+ private final ScheduledExecutorService executorService = Executors.newSingleThreadScheduledExecutor();
+ private final InternalMetricsCache cache;
+ static final String RAW_METRICS_CACHE = "RAW_METRICS_CACHE_INSTANCE";
+
+ public RawMetricsSource(int internalCacheInterval, ExternalMetricsSink rawMetricsSink) {
+ this.internalCacheInterval = internalCacheInterval;
+ this.rawMetricsSink = rawMetricsSink;
+ this.cache = InternalMetricsCacheProvider.getInstance().getCacheInstance(RAW_METRICS_CACHE);
+ if (rawMetricsSink.getFlushSeconds() > internalCacheInterval) {
+ initializeFixedRateScheduler();
+ }
+ }
+
+ @Override
+ public void publishTimelineMetrics(Collection<TimelineMetrics> metrics) {
+ // TODO: Adjust default flush to reasonable defaults > 3 seconds
+ if (rawMetricsSink.getFlushSeconds() > internalCacheInterval) {
+ // Need to cache only if external sink cannot keep up and thereby has
+ // different flush interval as compared to HBase flush
+ cache.putAll(metrics); // Scheduler initialized already for flush
+ } else {
+ submitDataWithTimeout(metrics);
+ }
+ }
+
+ private void initializeFixedRateScheduler() {
+ executorService.scheduleAtFixedRate(() -> rawMetricsSink.sinkMetricData(cache.evictAll()),
+ rawMetricsSink.getFlushSeconds(), rawMetricsSink.getFlushSeconds(), TimeUnit.SECONDS);
+ }
+
+ private void submitDataWithTimeout(final Collection<TimelineMetrics> metrics) {
+ Future f = executorService.submit(() -> {
+ rawMetricsSink.sinkMetricData(metrics);
+ return null;
+ });
+ try {
+ f.get(rawMetricsSink.getSinkTimeOutSeconds(), TimeUnit.SECONDS);
+ } catch (InterruptedException e) {
+ LOG.warn("Raw metrics sink interrupted.");
+ } catch (ExecutionException e) {
+ LOG.warn("Exception on sinking metrics", e);
+ } catch (TimeoutException e) {
+ LOG.warn("Timeout exception on sinking metrics", e);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/source/cache/InternalMetricCacheKey.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/source/cache/InternalMetricCacheKey.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/source/cache/InternalMetricCacheKey.java
new file mode 100644
index 0000000..bfb610f
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/source/cache/InternalMetricCacheKey.java
@@ -0,0 +1,109 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.metrics.core.timeline.source.cache;
+
+public class InternalMetricCacheKey {
+ private String metricName;
+ private String appId;
+ private String instanceId;
+ private String hostname;
+ private long startTime; // Useful for debugging
+
+ public InternalMetricCacheKey(String metricName, String appId, String instanceId, String hostname, long startTime) {
+ this.metricName = metricName;
+ this.appId = appId;
+ this.instanceId = instanceId;
+ this.hostname = hostname;
+ this.startTime = startTime;
+ }
+
+ public String getMetricName() {
+ return metricName;
+ }
+
+ public void setMetricName(String metricName) {
+ this.metricName = metricName;
+ }
+
+ public String getAppId() {
+ return appId;
+ }
+
+ public void setAppId(String appId) {
+ this.appId = appId;
+ }
+
+ public String getInstanceId() {
+ return instanceId;
+ }
+
+ public void setInstanceId(String instanceId) {
+ this.instanceId = instanceId;
+ }
+
+ public String getHostname() {
+ return hostname;
+ }
+
+ public void setHostname(String hostname) {
+ this.hostname = hostname;
+ }
+
+ public long getStartTime() {
+ return startTime;
+ }
+
+ public void setStartTime(long startTime) {
+ this.startTime = startTime;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+
+ InternalMetricCacheKey that = (InternalMetricCacheKey) o;
+
+ if (!getMetricName().equals(that.getMetricName())) return false;
+ if (!getAppId().equals(that.getAppId())) return false;
+ if (getInstanceId() != null ? !getInstanceId().equals(that.getInstanceId()) : that.getInstanceId() != null)
+ return false;
+ return getHostname() != null ? getHostname().equals(that.getHostname()) : that.getHostname() == null;
+
+ }
+
+ @Override
+ public int hashCode() {
+ int result = getMetricName().hashCode();
+ result = 31 * result + getAppId().hashCode();
+ result = 31 * result + (getInstanceId() != null ? getInstanceId().hashCode() : 0);
+ result = 31 * result + (getHostname() != null ? getHostname().hashCode() : 0);
+ return result;
+ }
+
+ @Override
+ public String toString() {
+ return "InternalMetricCacheKey{" +
+ "metricName='" + metricName + '\'' +
+ ", appId='" + appId + '\'' +
+ ", instanceId='" + instanceId + '\'' +
+ ", hostname='" + hostname + '\'' +
+ ", startTime=" + startTime +
+ '}';
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/source/cache/InternalMetricCacheValue.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/source/cache/InternalMetricCacheValue.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/source/cache/InternalMetricCacheValue.java
new file mode 100644
index 0000000..6573c4e
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/source/cache/InternalMetricCacheValue.java
@@ -0,0 +1,37 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.metrics.core.timeline.source.cache;
+
+import java.util.Map;
+import java.util.TreeMap;
+
+public class InternalMetricCacheValue {
+ private TreeMap<Long, Double> metricValues = new TreeMap<Long, Double>();
+
+ public TreeMap<Long, Double> getMetricValues() {
+ return metricValues;
+ }
+
+ public void setMetricValues(TreeMap<Long, Double> metricValues) {
+ this.metricValues = metricValues;
+ }
+
+ public void addMetricValues(Map<Long, Double> metricValues) {
+ this.metricValues.putAll(metricValues);
+ }
+}
[13/28] ambari git commit: AMBARI-22740 : Fix integration test for
HBase in branch-3.0-ams due to UUID changes. (avijayan)
Posted by av...@apache.org.
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/availability/MetricCollectorHAController.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/availability/MetricCollectorHAController.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/availability/MetricCollectorHAController.java
deleted file mode 100644
index d74f253..0000000
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/availability/MetricCollectorHAController.java
+++ /dev/null
@@ -1,307 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p/>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p/>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.availability;
-
-import static org.apache.helix.model.IdealState.RebalanceMode.FULL_AUTO;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-import java.util.TreeSet;
-import java.util.concurrent.Executors;
-import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.TimeUnit;
-
-import org.I0Itec.zkclient.exception.ZkNoNodeException;
-import org.apache.commons.collections.CollectionUtils;
-import org.apache.commons.lang.StringUtils;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.MetricsSystemInitializationException;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration;
-import org.apache.helix.HelixAdmin;
-import org.apache.helix.HelixException;
-import org.apache.helix.HelixManager;
-import org.apache.helix.HelixManagerFactory;
-import org.apache.helix.InstanceType;
-import org.apache.helix.NotificationContext;
-import org.apache.helix.controller.GenericHelixController;
-import org.apache.helix.manager.zk.ZKHelixAdmin;
-import org.apache.helix.model.ExternalView;
-import org.apache.helix.model.InstanceConfig;
-import org.apache.helix.model.LiveInstance;
-import org.apache.helix.model.OnlineOfflineSMD;
-import org.apache.helix.model.StateModelDefinition;
-import org.apache.helix.tools.StateModelConfigGenerator;
-
-import com.google.common.base.Joiner;
-
-;
-
-public class MetricCollectorHAController {
- private static final Log LOG = LogFactory.getLog(MetricCollectorHAController.class);
-
- static final String CLUSTER_NAME = "ambari-metrics-cluster";
- static final String METRIC_AGGREGATORS = "METRIC_AGGREGATORS";
- static final String DEFAULT_STATE_MODEL = OnlineOfflineSMD.name;
- static final String INSTANCE_NAME_DELIMITER = "_";
-
- final String zkConnectUrl;
- final String instanceHostname;
- final InstanceConfig instanceConfig;
- final AggregationTaskRunner aggregationTaskRunner;
- final TimelineMetricConfiguration configuration;
-
- // Cache list of known live instances
- final List<String> liveInstanceNames = new ArrayList<>();
-
- // Helix Admin
- HelixAdmin admin;
- // Helix Manager
- HelixManager manager;
-
- private volatile boolean isInitialized = false;
-
- public MetricCollectorHAController(TimelineMetricConfiguration configuration) {
- this.configuration = configuration;
- String instancePort;
- try {
- instanceHostname = configuration.getInstanceHostnameFromEnv();
- instancePort = configuration.getInstancePort();
-
- } catch (Exception e) {
- LOG.error("Error reading configs from classpath, will resort to defaults.", e);
- throw new MetricsSystemInitializationException(e.getMessage());
- }
-
- try {
- String zkClientPort = configuration.getClusterZKClientPort();
- String zkQuorum = configuration.getClusterZKQuorum();
-
- if (StringUtils.isEmpty(zkClientPort) || StringUtils.isEmpty(zkQuorum)) {
- throw new Exception("Unable to parse zookeeper quorum. clientPort = "
- + zkClientPort +", quorum = " + zkQuorum);
- }
-
- zkConnectUrl = configuration.getZkConnectionUrl(zkClientPort, zkQuorum);
-
- } catch (Exception e) {
- LOG.error("Unable to load hbase-site from classpath.", e);
- throw new MetricsSystemInitializationException(e.getMessage());
- }
-
- instanceConfig = new InstanceConfig(instanceHostname + INSTANCE_NAME_DELIMITER + instancePort);
- instanceConfig.setHostName(instanceHostname);
- instanceConfig.setPort(instancePort);
- instanceConfig.setInstanceEnabled(true);
- aggregationTaskRunner = new AggregationTaskRunner(
- instanceConfig.getInstanceName(), zkConnectUrl, getClusterName());
- }
-
- /**
- * Name of Helix znode
- */
- public String getClusterName() {
- return CLUSTER_NAME;
- }
-
- /**
- * Initialize the instance with zookeeper via Helix
- */
- public void initializeHAController() throws Exception {
- String clusterName = getClusterName();
- admin = new ZKHelixAdmin(zkConnectUrl);
- // create cluster
- LOG.info("Creating zookeeper cluster node: " + clusterName);
- boolean clusterAdded = admin.addCluster(clusterName, false);
- LOG.info("Was cluster added successfully? " + clusterAdded);
-
- // Adding host to the cluster
- boolean success = false;
- int tries = 5;
- int sleepTimeInSeconds = 5;
-
- for (int i = 0; i < tries && !success; i++) {
- try {
- List<String> nodes = admin.getInstancesInCluster(clusterName);
- if (CollectionUtils.isEmpty(nodes) || !nodes.contains(instanceConfig.getInstanceName())) {
- LOG.info("Adding participant instance " + instanceConfig);
- admin.addInstance(clusterName, instanceConfig);
- }
- success = true;
- } catch (HelixException | ZkNoNodeException ex) {
- LOG.warn("Helix Cluster not yet setup fully.");
- if (i < tries - 1) {
- LOG.info("Waiting for " + sleepTimeInSeconds + " seconds and retrying.");
- TimeUnit.SECONDS.sleep(sleepTimeInSeconds);
- } else {
- LOG.error(ex);
- }
- }
- }
-
- if (!success) {
- LOG.info("Trying to create " + clusterName + " again since waiting for the creation did not help.");
- admin.addCluster(clusterName, true);
- List<String> nodes = admin.getInstancesInCluster(clusterName);
- if (CollectionUtils.isEmpty(nodes) || !nodes.contains(instanceConfig.getInstanceName())) {
- LOG.info("Adding participant instance " + instanceConfig);
- admin.addInstance(clusterName, instanceConfig);
- }
- }
-
- // Add a state model
- if (admin.getStateModelDef(clusterName, DEFAULT_STATE_MODEL) == null) {
- LOG.info("Adding ONLINE-OFFLINE state model to the cluster");
- admin.addStateModelDef(clusterName, DEFAULT_STATE_MODEL, new StateModelDefinition(
- StateModelConfigGenerator.generateConfigForOnlineOffline()));
- }
-
- // Add resources with 1 cluster-wide replica
- // Since our aggregators are unbalanced in terms of work distribution we
- // only need to distribute writes to METRIC_AGGREGATE and
- // METRIC_RECORD_MINUTE
- List<String> resources = admin.getResourcesInCluster(clusterName);
- if (!resources.contains(METRIC_AGGREGATORS)) {
- LOG.info("Adding resource " + METRIC_AGGREGATORS + " with 2 partitions and 1 replicas");
- admin.addResource(clusterName, METRIC_AGGREGATORS, 2, DEFAULT_STATE_MODEL, FULL_AUTO.toString());
- }
- // this will set up the ideal state, it calculates the preference list for
- // each partition similar to consistent hashing
- admin.rebalance(clusterName, METRIC_AGGREGATORS, 1);
-
- // Start participant
- startAggregators();
-
- // Start controller
- startController();
-
- Runtime.getRuntime().addShutdownHook(new Thread() {
- @Override
- public void run() {
- aggregationTaskRunner.stop();
- manager.disconnect();
- }
- });
-
- isInitialized = true;
- }
-
- /**
- * Return true if HA controller is enabled.
- */
- public boolean isInitialized() {
- return isInitialized;
- }
-
- private void startAggregators() {
- try {
- aggregationTaskRunner.initialize();
-
- } catch (Exception e) {
- LOG.error("Unable to start aggregators.", e);
- throw new MetricsSystemInitializationException(e.getMessage());
- }
- }
-
- private void startController() throws Exception {
- manager = HelixManagerFactory.getZKHelixManager(
- getClusterName(),
- instanceHostname,
- InstanceType.CONTROLLER,
- zkConnectUrl
- );
-
- manager.connect();
- HelixController controller = new HelixController();
- manager.addLiveInstanceChangeListener(controller);
- }
-
- public AggregationTaskRunner getAggregationTaskRunner() {
- return aggregationTaskRunner;
- }
-
- public List<String> getLiveInstanceHostNames() {
- List<String> liveInstanceHostNames = new ArrayList<>();
-
- for (String instance : liveInstanceNames) {
- liveInstanceHostNames.add(instance.split(INSTANCE_NAME_DELIMITER)[0]);
- }
-
- return liveInstanceHostNames;
- }
-
- public class HelixController extends GenericHelixController {
- ScheduledExecutorService executorService = Executors.newSingleThreadScheduledExecutor();
- Joiner joiner = Joiner.on(", ").skipNulls();
-
- @Override
- public void onLiveInstanceChange(List<LiveInstance> liveInstances, NotificationContext changeContext) {
- super.onLiveInstanceChange(liveInstances, changeContext);
-
- liveInstanceNames.clear();
- for (LiveInstance instance : liveInstances) {
- liveInstanceNames.add(instance.getInstanceName());
- }
-
- LOG.info("Detected change in liveliness of Collector instances. " +
- "LiveIsntances = " + joiner.join(liveInstanceNames));
- // Print HA state - after some delay
- executorService.schedule(new Runnable() {
- @Override
- public void run() {
- printClusterState();
- }
- }, 30, TimeUnit.SECONDS);
-
-
- }
- }
-
- public void printClusterState() {
- StringBuilder sb = new StringBuilder("\n######################### Cluster HA state ########################");
-
- ExternalView resourceExternalView = admin.getResourceExternalView(getClusterName(), METRIC_AGGREGATORS);
- if (resourceExternalView != null) {
- getPrintableResourceState(resourceExternalView, METRIC_AGGREGATORS, sb);
- }
- sb.append("\n##################################################");
- LOG.info(sb.toString());
- }
-
- private void getPrintableResourceState(ExternalView resourceExternalView,
- String resourceName,
- StringBuilder sb) {
- TreeSet<String> sortedSet = new TreeSet<>(resourceExternalView.getPartitionSet());
- sb.append("\nCLUSTER: ");
- sb.append(getClusterName());
- sb.append("\nRESOURCE: ");
- sb.append(resourceName);
- for (String partitionName : sortedSet) {
- sb.append("\nPARTITION: ");
- sb.append(partitionName).append("\t");
- Map<String, String> states = resourceExternalView.getStateMap(partitionName);
- for (Map.Entry<String, String> stateEntry : states.entrySet()) {
- sb.append("\t");
- sb.append(stateEntry.getKey());
- sb.append("\t");
- sb.append(stateEntry.getValue());
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/availability/OnlineOfflineStateModelFactory.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/availability/OnlineOfflineStateModelFactory.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/availability/OnlineOfflineStateModelFactory.java
deleted file mode 100644
index a53dc3b..0000000
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/availability/OnlineOfflineStateModelFactory.java
+++ /dev/null
@@ -1,69 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p/>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p/>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.availability;
-
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.availability.AggregationTaskRunner.PARTITION_AGGREGATION_TYPES;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.aggregators.TimelineMetricAggregator.AGGREGATOR_TYPE;
-import org.apache.helix.NotificationContext;
-import org.apache.helix.model.Message;
-import org.apache.helix.participant.statemachine.StateModel;
-import org.apache.helix.participant.statemachine.StateModelFactory;
-
-public class OnlineOfflineStateModelFactory extends StateModelFactory<StateModel> {
- private static final Log LOG = LogFactory.getLog(OnlineOfflineStateModelFactory.class);
- private final String instanceName;
- private final AggregationTaskRunner taskRunner;
-
- public OnlineOfflineStateModelFactory(String instanceName, AggregationTaskRunner taskRunner) {
- this.instanceName = instanceName;
- this.taskRunner = taskRunner;
- }
-
- @Override
- public StateModel createNewStateModel(String resourceName, String partition) {
- LOG.info("Received request to process partition = " + partition + ", for " +
- "resource = " + resourceName + ", at " + instanceName);
- return new OnlineOfflineStateModel();
- }
-
- public class OnlineOfflineStateModel extends StateModel {
- public void onBecomeOnlineFromOffline(Message message, NotificationContext context) {
- String partitionName = message.getPartitionName();
- LOG.info("Received transition to Online from Offline for partition: " + partitionName);
- AGGREGATOR_TYPE type = PARTITION_AGGREGATION_TYPES.get(partitionName);
- taskRunner.setPartitionAggregationFunction(type);
- }
-
- public void onBecomeOfflineFromOnline(Message message, NotificationContext context) {
- String partitionName = message.getPartitionName();
- LOG.info("Received transition to Offline from Online for partition: " + partitionName);
- AGGREGATOR_TYPE type = PARTITION_AGGREGATION_TYPES.get(partitionName);
- taskRunner.unsetPartitionAggregationFunction(type);
- }
-
- public void onBecomeDroppedFromOffline(Message message, NotificationContext context) {
- String partitionName = message.getPartitionName();
- LOG.info("Received transition to Dropped from Offline for partition: " + partitionName);
- AGGREGATOR_TYPE type = PARTITION_AGGREGATION_TYPES.get(partitionName);
- taskRunner.unsetPartitionAggregationFunction(type);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/discovery/TimelineMetricHostMetadata.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/discovery/TimelineMetricHostMetadata.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/discovery/TimelineMetricHostMetadata.java
deleted file mode 100644
index 37c6394..0000000
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/discovery/TimelineMetricHostMetadata.java
+++ /dev/null
@@ -1,60 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p/>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p/>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.discovery;
-
-import java.util.Set;
-import java.util.concurrent.ConcurrentHashMap;
-
-public class TimelineMetricHostMetadata {
- //need concurrent data structure, only keys are used.
- private ConcurrentHashMap<String, String> hostedApps = new ConcurrentHashMap<>();
- private byte[] uuid;
-
- // Default constructor
- public TimelineMetricHostMetadata() {
- }
-
- public TimelineMetricHostMetadata(ConcurrentHashMap<String, String> hostedApps) {
- this.hostedApps = hostedApps;
- }
-
- public TimelineMetricHostMetadata(Set<String> hostedApps) {
- ConcurrentHashMap<String, String> appIdsMap = new ConcurrentHashMap<>();
- for (String appId : hostedApps) {
- appIdsMap.put(appId, appId);
- }
- this.hostedApps = appIdsMap;
- }
-
- public ConcurrentHashMap<String, String> getHostedApps() {
- return hostedApps;
- }
-
- public void setHostedApps(ConcurrentHashMap<String, String> hostedApps) {
- this.hostedApps = hostedApps;
- }
-
- public byte[] getUuid() {
- return uuid;
- }
-
- public void setUuid(byte[] uuid) {
- this.uuid = uuid;
- }
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/discovery/TimelineMetricMetadataKey.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/discovery/TimelineMetricMetadataKey.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/discovery/TimelineMetricMetadataKey.java
deleted file mode 100644
index 0c0ee5b..0000000
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/discovery/TimelineMetricMetadataKey.java
+++ /dev/null
@@ -1,80 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p/>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p/>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.discovery;
-
-import javax.xml.bind.annotation.XmlRootElement;
-
-import org.apache.commons.lang3.StringUtils;
-
-@XmlRootElement
-public class TimelineMetricMetadataKey {
- String metricName;
- String appId;
- String instanceId;
-
- public TimelineMetricMetadataKey(String metricName, String appId, String instanceId) {
- this.metricName = metricName;
- this.appId = appId;
- this.instanceId = instanceId;
- }
-
- public String getMetricName() {
- return metricName;
- }
-
- public String getAppId() {
- return appId;
- }
-
- public String getInstanceId() {
- return instanceId;
- }
-
- public void setAppId(String appId) {
- this.appId = appId;
- }
-
- @Override
- public boolean equals(Object o) {
- if (this == o) return true;
- if (o == null || getClass() != o.getClass()) return false;
-
- TimelineMetricMetadataKey that = (TimelineMetricMetadataKey) o;
-
- if (!metricName.equals(that.metricName)) return false;
- if (!appId.equals(that.appId)) return false;
- return (StringUtils.isNotEmpty(instanceId) ? instanceId.equals(that.instanceId) : StringUtils.isEmpty(that.instanceId));
- }
-
- @Override
- public int hashCode() {
- int result = metricName.hashCode();
- result = 31 * result + (appId != null ? appId.hashCode() : 0);
- result = 31 * result + (instanceId != null ? instanceId.hashCode() : 0);
- return result;
- }
-
- @Override
- public String toString() {
- return "TimelineMetricMetadataKey{" +
- "metricName='" + metricName + '\'' +
- ", appId='" + appId + '\'' +
- ", instanceId='" + instanceId + '\'' +
- '}';
- }
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/discovery/TimelineMetricMetadataManager.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/discovery/TimelineMetricMetadataManager.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/discovery/TimelineMetricMetadataManager.java
deleted file mode 100644
index beac866..0000000
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/discovery/TimelineMetricMetadataManager.java
+++ /dev/null
@@ -1,612 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p/>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p/>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.discovery;
-
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.DISABLE_METRIC_METADATA_MGMT;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.METRICS_METADATA_SYNC_INIT_DELAY;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.METRICS_METADATA_SYNC_SCHEDULE_DELAY;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.TIMELINE_METRICS_UUID_GEN_STRATEGY;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.TIMELINE_METRIC_METADATA_FILTERS;
-
-import java.net.MalformedURLException;
-import java.net.URISyntaxException;
-import java.sql.SQLException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.Executors;
-import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicBoolean;
-
-import org.apache.commons.collections.CollectionUtils;
-import org.apache.commons.collections.MapUtils;
-import org.apache.commons.lang.ArrayUtils;
-import org.apache.commons.lang.StringUtils;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.metrics2.sink.timeline.MetadataException;
-import org.apache.hadoop.metrics2.sink.timeline.TimelineMetric;
-import org.apache.hadoop.metrics2.sink.timeline.TimelineMetricMetadata;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.PhoenixHBaseAccessor;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.aggregators.TimelineClusterMetric;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.uuid.HashBasedUuidGenStrategy;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.uuid.MetricUuidGenStrategy;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.uuid.RandomUuidGenStrategy;
-
-public class TimelineMetricMetadataManager {
- private static final Log LOG = LogFactory.getLog(TimelineMetricMetadataManager.class);
- private boolean isDisabled = false;
- // Cache all metadata on retrieval
- private final Map<TimelineMetricMetadataKey, TimelineMetricMetadata> METADATA_CACHE = new ConcurrentHashMap<>();
- private final Map<String, TimelineMetricMetadataKey> uuidKeyMap = new ConcurrentHashMap<>();
- // Map to lookup apps on a host
- private final Map<String, TimelineMetricHostMetadata> HOSTED_APPS_MAP = new ConcurrentHashMap<>();
- private final Map<String, String> uuidHostMap = new ConcurrentHashMap<>();
- private final Map<String, Set<String>> INSTANCE_HOST_MAP = new ConcurrentHashMap<>();
- // Sync only when needed
- AtomicBoolean SYNC_HOSTED_APPS_METADATA = new AtomicBoolean(false);
- AtomicBoolean SYNC_HOSTED_INSTANCES_METADATA = new AtomicBoolean(false);
- private MetricUuidGenStrategy uuidGenStrategy = new HashBasedUuidGenStrategy();
- public static final int TIMELINE_METRIC_UUID_LENGTH = 16;
- public static final int HOSTNAME_UUID_LENGTH = 4;
-
- // Single thread to sync back new writes to the store
- private final ScheduledExecutorService executorService = Executors.newSingleThreadScheduledExecutor();
-
- private PhoenixHBaseAccessor hBaseAccessor;
- private Configuration metricsConf;
-
- TimelineMetricMetadataSync metricMetadataSync;
- // Filter metrics names matching given patterns, from metadata
- final List<String> metricNameFilters = new ArrayList<>();
-
- // Test friendly construction since mock instrumentation is difficult to get
- // working with hadoop mini cluster
- public TimelineMetricMetadataManager(Configuration metricsConf, PhoenixHBaseAccessor hBaseAccessor) {
- this.metricsConf = metricsConf;
- this.hBaseAccessor = hBaseAccessor;
- String patternStrings = metricsConf.get(TIMELINE_METRIC_METADATA_FILTERS);
- if (!StringUtils.isEmpty(patternStrings)) {
- metricNameFilters.addAll(Arrays.asList(patternStrings.split(",")));
- }
-
- uuidGenStrategy = getUuidStrategy(metricsConf);
- }
-
- public TimelineMetricMetadataManager(PhoenixHBaseAccessor hBaseAccessor) throws MalformedURLException, URISyntaxException {
- this(TimelineMetricConfiguration.getInstance().getMetricsConf(), hBaseAccessor);
- }
-
- /**
- * Initialize Metadata from the store
- */
- public void initializeMetadata() {
- if (metricsConf.getBoolean(DISABLE_METRIC_METADATA_MGMT, false)) {
- isDisabled = true;
- } else {
- metricMetadataSync = new TimelineMetricMetadataSync(this);
- // Schedule the executor to sync to store
- executorService.scheduleWithFixedDelay(metricMetadataSync,
- metricsConf.getInt(METRICS_METADATA_SYNC_INIT_DELAY, 120), // 2 minutes
- metricsConf.getInt(METRICS_METADATA_SYNC_SCHEDULE_DELAY, 300), // 5 minutes
- TimeUnit.SECONDS);
- // Read from store and initialize map
- try {
- Map<TimelineMetricMetadataKey, TimelineMetricMetadata> metadata = getMetadataFromStore();
-
- LOG.info("Retrieved " + metadata.size() + ", metadata objects from store.");
- // Store in the cache
- METADATA_CACHE.putAll(metadata);
-
- Map<String, TimelineMetricHostMetadata> hostedAppData = getHostedAppsFromStore();
-
- LOG.info("Retrieved " + hostedAppData.size() + " host objects from store.");
- HOSTED_APPS_MAP.putAll(hostedAppData);
-
- loadUuidMapsOnInit();
-
- hBaseAccessor.setMetadataInstance(this);
- } catch (SQLException e) {
- LOG.warn("Exception loading metric metadata", e);
- }
- }
- }
-
- public Map<TimelineMetricMetadataKey, TimelineMetricMetadata> getMetadataCache() {
- return METADATA_CACHE;
- }
-
- public TimelineMetricMetadata getMetadataCacheValue(TimelineMetricMetadataKey key) {
- return METADATA_CACHE.get(key);
- }
-
- public Map<String, TimelineMetricHostMetadata> getHostedAppsCache() {
- return HOSTED_APPS_MAP;
- }
-
- public Map<String, Set<String>> getHostedInstanceCache() {
- return INSTANCE_HOST_MAP;
- }
-
- public boolean syncHostedAppsMetadata() {
- return SYNC_HOSTED_APPS_METADATA.get();
- }
-
- public boolean syncHostedInstanceMetadata() {
- return SYNC_HOSTED_INSTANCES_METADATA.get();
- }
-
- public void markSuccessOnSyncHostedAppsMetadata() {
- SYNC_HOSTED_APPS_METADATA.set(false);
- }
-
- public void markSuccessOnSyncHostedInstanceMetadata() {
- SYNC_HOSTED_INSTANCES_METADATA.set(false);
- }
- /**
- * Test metric name for valid patterns and return true/false
- */
- boolean skipMetadataCache(String metricName) {
- for (String pattern : metricNameFilters) {
- if (metricName.contains(pattern)) {
- return true;
- }
- }
- return false;
- }
-
- /**
- * Update value in metadata cache
- * @param metadata @TimelineMetricMetadata
- */
- public void putIfModifiedTimelineMetricMetadata(TimelineMetricMetadata metadata) {
- if (skipMetadataCache(metadata.getMetricName())) {
- return;
- }
-
- TimelineMetricMetadataKey key = new TimelineMetricMetadataKey(
- metadata.getMetricName(), metadata.getAppId(), metadata.getInstanceId());
-
- TimelineMetricMetadata metadataFromCache = METADATA_CACHE.get(key);
-
- if (metadataFromCache != null) {
- try {
- if (metadataFromCache.needsToBeSynced(metadata)) {
- metadata.setIsPersisted(false); // Set the flag to ensure sync to store on next run
- METADATA_CACHE.put(key, metadata);
- }
- } catch (MetadataException e) {
- LOG.warn("Error inserting Metadata in cache.", e);
- }
-
- } else {
- METADATA_CACHE.put(key, metadata);
- }
- }
-
- /**
- * Update value in hosted apps cache
- * @param hostname Host name
- * @param appId Application Id
- */
- public void putIfModifiedHostedAppsMetadata(String hostname, String appId) {
- TimelineMetricHostMetadata timelineMetricHostMetadata = HOSTED_APPS_MAP.get(hostname);
- ConcurrentHashMap<String, String> apps = (timelineMetricHostMetadata != null) ? timelineMetricHostMetadata.getHostedApps() : null;
- if (apps == null) {
- apps = new ConcurrentHashMap<>();
- if (timelineMetricHostMetadata == null) {
- HOSTED_APPS_MAP.put(hostname, new TimelineMetricHostMetadata(apps));
- } else {
- HOSTED_APPS_MAP.get(hostname).setHostedApps(apps);
- }
- }
-
- if (!apps.containsKey(appId)) {
- apps.put(appId, appId);
- SYNC_HOSTED_APPS_METADATA.set(true);
- }
- }
-
- public void putIfModifiedHostedInstanceMetadata(String instanceId, String hostname) {
- if (StringUtils.isEmpty(instanceId)) {
- return;
- }
-
- Set<String> hosts = INSTANCE_HOST_MAP.get(instanceId);
- if (hosts == null) {
- hosts = new HashSet<>();
- INSTANCE_HOST_MAP.put(instanceId, hosts);
- }
-
- if (!hosts.contains(hostname)) {
- hosts.add(hostname);
- SYNC_HOSTED_INSTANCES_METADATA.set(true);
- }
- }
-
- public void persistMetadata(Collection<TimelineMetricMetadata> metadata) throws SQLException {
- hBaseAccessor.saveMetricMetadata(metadata);
- }
-
- public void persistHostedAppsMetadata(Map<String, TimelineMetricHostMetadata> hostedApps) throws SQLException {
- hBaseAccessor.saveHostAppsMetadata(hostedApps);
- }
-
- public void persistHostedInstanceMetadata(Map<String, Set<String>> hostedInstancesMetadata) throws SQLException {
- hBaseAccessor.saveInstanceHostsMetadata(hostedInstancesMetadata);
- }
-
- public TimelineMetricMetadata getTimelineMetricMetadata(TimelineMetric timelineMetric, boolean isWhitelisted) {
- return new TimelineMetricMetadata(
- timelineMetric.getMetricName(),
- timelineMetric.getAppId(),
- timelineMetric.getInstanceId(),
- timelineMetric.getUnits(),
- timelineMetric.getType(),
- timelineMetric.getStartTime(),
- supportAggregates(timelineMetric),
- isWhitelisted
- );
- }
-
- public boolean isDisabled() {
- return isDisabled;
- }
-
- boolean isDistributedModeEnabled() {
- return metricsConf.get("timeline.metrics.service.operation.mode").equals("distributed");
- }
-
- /**
- * Fetch metrics metadata from store
- * @throws SQLException
- */
- Map<TimelineMetricMetadataKey, TimelineMetricMetadata> getMetadataFromStore() throws SQLException {
- return hBaseAccessor.getTimelineMetricMetadata();
- }
-
- /**
- * Fetch hosted apps from store
- * @throws SQLException
- */
- Map<String, TimelineMetricHostMetadata> getHostedAppsFromStore() throws SQLException {
- return hBaseAccessor.getHostedAppsMetadata();
- }
-
- Map<String, Set<String>> getHostedInstancesFromStore() throws SQLException {
- return hBaseAccessor.getInstanceHostsMetdata();
- }
-
- private boolean supportAggregates(TimelineMetric metric) {
- return MapUtils.isEmpty(metric.getMetadata()) ||
- !(String.valueOf(true).equals(metric.getMetadata().get("skipAggregation")));
- }
-
- //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
- // UUID Management
- //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-
-
- /**
- * Load the UUID mappings from the UUID table on startup.
- */
- private void loadUuidMapsOnInit() {
-
- for (TimelineMetricMetadataKey key : METADATA_CACHE.keySet()) {
- TimelineMetricMetadata timelineMetricMetadata = METADATA_CACHE.get(key);
- if (timelineMetricMetadata != null && timelineMetricMetadata.getUuid() != null) {
- uuidKeyMap.put(new String(timelineMetricMetadata.getUuid()), key);
- }
- }
-
- for (String host : HOSTED_APPS_MAP.keySet()) {
- TimelineMetricHostMetadata timelineMetricHostMetadata = HOSTED_APPS_MAP.get(host);
- if (timelineMetricHostMetadata != null && timelineMetricHostMetadata.getUuid() != null) {
- uuidHostMap.put(new String(timelineMetricHostMetadata.getUuid()), host);
- }
- }
- }
-
- /**
- * Returns the UUID gen strategy.
- * @param configuration
- * @return
- */
- private MetricUuidGenStrategy getUuidStrategy(Configuration configuration) {
- String strategy = configuration.get(TIMELINE_METRICS_UUID_GEN_STRATEGY, "");
- if ("random".equalsIgnoreCase(strategy)) {
- return new RandomUuidGenStrategy();
- } else {
- return new HashBasedUuidGenStrategy();
- }
- }
-
- /**
- * Given the hostname, generates a byte array of length 'HOSTNAME_UUID_LENGTH'
- * @param hostname
- * @return uuid byte array of length 'HOSTNAME_UUID_LENGTH'
- */
- private byte[] getUuidForHostname(String hostname) {
-
- TimelineMetricHostMetadata timelineMetricHostMetadata = HOSTED_APPS_MAP.get(hostname);
- if (timelineMetricHostMetadata != null) {
- byte[] uuid = timelineMetricHostMetadata.getUuid();
- if (uuid != null) {
- return uuid;
- }
- }
-
- byte[] uuid = uuidGenStrategy.computeUuid(hostname, HOSTNAME_UUID_LENGTH);
-
- String uuidStr = new String(uuid);
- if (uuidHostMap.containsKey(uuidStr)) {
- //TODO fix the collisions
- LOG.error("Duplicate key computed for " + hostname +", Collides with " + uuidHostMap.get(uuidStr));
- return uuid;
- }
-
- if (timelineMetricHostMetadata == null) {
- timelineMetricHostMetadata = new TimelineMetricHostMetadata();
- HOSTED_APPS_MAP.put(hostname, timelineMetricHostMetadata);
- }
- timelineMetricHostMetadata.setUuid(uuid);
- uuidHostMap.put(uuidStr, hostname);
-
- return uuid;
- }
-
- /**
- * Given a timelineClusterMetric instance, generates a UUID for Metric-App-Instance combination.
- * @param timelineClusterMetric
- * @return uuid byte array of length 'TIMELINE_METRIC_UUID_LENGTH'
- */
- public byte[] getUuid(TimelineClusterMetric timelineClusterMetric) {
- TimelineMetricMetadataKey key = new TimelineMetricMetadataKey(timelineClusterMetric.getMetricName(),
- timelineClusterMetric.getAppId(), timelineClusterMetric.getInstanceId());
-
- TimelineMetricMetadata timelineMetricMetadata = METADATA_CACHE.get(key);
- if (timelineMetricMetadata != null) {
- byte[] uuid = timelineMetricMetadata.getUuid();
- if (uuid != null) {
- return uuid;
- }
- }
-
- byte[] uuid = uuidGenStrategy.computeUuid(timelineClusterMetric, TIMELINE_METRIC_UUID_LENGTH);
-
- String uuidStr = new String(uuid);
- if (uuidKeyMap.containsKey(uuidStr) && !uuidKeyMap.get(uuidStr).equals(key)) {
- TimelineMetricMetadataKey collidingKey = (TimelineMetricMetadataKey)uuidKeyMap.get(uuidStr);
- //TODO fix the collisions
- /**
- * 2017-08-23 14:12:35,922 ERROR org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.discovery.TimelineMetricMetadataManager:
- * Duplicate key [52, 50, 51, 53, 50, 53, 53, 53, 49, 54, 57, 50, 50, 54, 0, 0]([B@278a93f9) computed for
- * TimelineClusterMetric{metricName='sdisk_dm-11_write_count', appId='hbase', instanceId='', timestamp=1503497400000}, Collides with
- * TimelineMetricMetadataKey{metricName='sdisk_dm-20_write_count', appId='hbase', instanceId=''}
- */
- LOG.error("Duplicate key " + Arrays.toString(uuid) + "(" + uuid + ") computed for " + timelineClusterMetric.toString() + ", Collides with " + collidingKey.toString());
- return uuid;
- }
-
- if (timelineMetricMetadata == null) {
- timelineMetricMetadata = new TimelineMetricMetadata();
- timelineMetricMetadata.setMetricName(timelineClusterMetric.getMetricName());
- timelineMetricMetadata.setAppId(timelineClusterMetric.getAppId());
- timelineMetricMetadata.setInstanceId(timelineClusterMetric.getInstanceId());
- METADATA_CACHE.put(key, timelineMetricMetadata);
- }
-
- timelineMetricMetadata.setUuid(uuid);
- timelineMetricMetadata.setIsPersisted(false);
- uuidKeyMap.put(uuidStr, key);
- return uuid;
- }
-
- /**
- * Given a timelineMetric instance, generates a UUID for Metric-App-Instance combination.
- * @param timelineMetric
- * @return uuid byte array of length 'TIMELINE_METRIC_UUID_LENGTH' + 'HOSTNAME_UUID_LENGTH'
- */
- public byte[] getUuid(TimelineMetric timelineMetric) {
-
- byte[] metricUuid = getUuid(new TimelineClusterMetric(timelineMetric.getMetricName(), timelineMetric.getAppId(),
- timelineMetric.getInstanceId(), -1l));
- byte[] hostUuid = getUuidForHostname(timelineMetric.getHostName());
-
- return ArrayUtils.addAll(metricUuid, hostUuid);
- }
-
- public byte[] getUuid(String metricName, String appId, String instanceId, String hostname) {
-
- byte[] metricUuid = getUuid(new TimelineClusterMetric(metricName, appId, instanceId, -1l));
- if (StringUtils.isNotEmpty(hostname)) {
- byte[] hostUuid = getUuidForHostname(hostname);
- return ArrayUtils.addAll(metricUuid, hostUuid);
- }
- return metricUuid;
- }
-
- public String getMetricNameFromUuid(byte[] uuid) {
-
- byte[] metricUuid = uuid;
- if (uuid.length == TIMELINE_METRIC_UUID_LENGTH + HOSTNAME_UUID_LENGTH) {
- metricUuid = ArrayUtils.subarray(uuid, 0, TIMELINE_METRIC_UUID_LENGTH);
- }
-
- TimelineMetricMetadataKey key = uuidKeyMap.get(new String(metricUuid));
- return key != null ? key.getMetricName() : null;
- }
-
- public TimelineMetric getMetricFromUuid(byte[] uuid) {
- if (uuid == null) {
- return null;
- }
-
- if (uuid.length == TIMELINE_METRIC_UUID_LENGTH) {
- TimelineMetricMetadataKey key = uuidKeyMap.get(new String(uuid));
- return key != null ? new TimelineMetric(key.metricName, null, key.appId, key.instanceId) : null;
- } else {
- byte[] metricUuid = ArrayUtils.subarray(uuid, 0, TIMELINE_METRIC_UUID_LENGTH);
- TimelineMetricMetadataKey key = uuidKeyMap.get(new String(metricUuid));
- if (key == null) {
- LOG.error("TimelineMetricMetadataKey is null for : " + Arrays.toString(uuid));
- return null;
- }
- TimelineMetric timelineMetric = new TimelineMetric();
- timelineMetric.setMetricName(key.metricName);
- timelineMetric.setAppId(key.appId);
- timelineMetric.setInstanceId(key.instanceId);
-
- byte[] hostUuid = ArrayUtils.subarray(uuid, TIMELINE_METRIC_UUID_LENGTH, HOSTNAME_UUID_LENGTH + TIMELINE_METRIC_UUID_LENGTH);
- timelineMetric.setHostName(uuidHostMap.get(new String(hostUuid)));
- return timelineMetric;
- }
- }
-
- /**
- * Returns the set of UUIDs for a given GET request. If there are wildcards (%), resolves them based on UUID map.
- * @param metricNames
- * @param hostnames
- * @param appId
- * @param instanceId
- * @return Set of UUIds
- */
- public List<byte[]> getUuids(Collection<String> metricNames, List<String> hostnames, String appId, String instanceId) {
-
- Collection<String> sanitizedMetricNames = new HashSet<>();
-
- for (String metricName : metricNames) {
- if (metricName.contains("%")) {
- String metricRegEx;
- //Special case handling for metric name with * and __%.
- //For example, dfs.NNTopUserOpCounts.windowMs=300000.op=*.user=%.count
- // or dfs.NNTopUserOpCounts.windowMs=300000.op=__%.user=%.count
- if (metricName.contains("*") || metricName.contains("__%")) {
- String metricNameWithEscSeq = metricName.replace("*", "\\*").replace("__%", "..%");
- metricRegEx = metricNameWithEscSeq.replace("%", ".*");
- } else {
- metricRegEx = metricName.replace("%", ".*");
- }
- for (TimelineMetricMetadataKey key : METADATA_CACHE.keySet()) {
- String metricNameFromMetadata = key.getMetricName();
- if (metricNameFromMetadata.matches(metricRegEx)) {
- sanitizedMetricNames.add(metricNameFromMetadata);
- }
- }
- } else {
- sanitizedMetricNames.add(metricName);
- }
- }
-
- Set<String> sanitizedHostNames = new HashSet<>();
- if (CollectionUtils.isNotEmpty(hostnames)) {
- for (String hostname : hostnames) {
- if (hostname.contains("%")) {
- String hostRegEx;
- hostRegEx = hostname.replace("%", ".*");
- for (String host : HOSTED_APPS_MAP.keySet()) {
- if (host.matches(hostRegEx)) {
- sanitizedHostNames.add(host);
- }
- }
- } else {
- sanitizedHostNames.add(hostname);
- }
- }
- }
-
- List<byte[]> uuids = new ArrayList<>();
-
- if (!(appId.equals("HOST") || appId.equals("FLUME_HANDLER"))) { //HACK.. Why??
- appId = appId.toLowerCase();
- }
- if (CollectionUtils.isNotEmpty(sanitizedHostNames)) {
- if (CollectionUtils.isNotEmpty(sanitizedMetricNames)) {
- for (String metricName : sanitizedMetricNames) {
- TimelineMetric metric = new TimelineMetric();
- metric.setMetricName(metricName);
- metric.setAppId(appId);
- metric.setInstanceId(instanceId);
- for (String hostname : sanitizedHostNames) {
- metric.setHostName(hostname);
- byte[] uuid = getUuid(metric);
- if (uuid != null) {
- uuids.add(uuid);
- }
- }
- }
- } else {
- for (String hostname : sanitizedHostNames) {
- byte[] uuid = getUuidForHostname(hostname);
- if (uuid != null) {
- uuids.add(uuid);
- }
- }
- }
- } else {
- for (String metricName : sanitizedMetricNames) {
- TimelineClusterMetric metric = new TimelineClusterMetric(metricName, appId, instanceId, -1l);
- byte[] uuid = getUuid(metric);
- if (uuid != null) {
- uuids.add(uuid);
- }
- }
- }
-
- return uuids;
- }
-
- public Map<String, TimelineMetricMetadataKey> getUuidKeyMap() {
- return uuidKeyMap;
- }
-
- public List<String> getNotLikeHostnames(List<String> hostnames) {
- List<String> result = new ArrayList<>();
- Set<String> sanitizedHostNames = new HashSet<>();
- if (CollectionUtils.isNotEmpty(hostnames)) {
- for (String hostname : hostnames) {
- if (hostname.contains("%")) {
- String hostRegEx;
- hostRegEx = hostname.replace("%", ".*");
- for (String host : HOSTED_APPS_MAP.keySet()) {
- if (host.matches(hostRegEx)) {
- sanitizedHostNames.add(host);
- }
- }
- } else {
- sanitizedHostNames.add(hostname);
- }
- }
- }
-
- for (String hostname: HOSTED_APPS_MAP.keySet()) {
- if (!sanitizedHostNames.contains(hostname)) {
- result.add(hostname);
- }
- }
- return result;
- }
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/discovery/TimelineMetricMetadataSync.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/discovery/TimelineMetricMetadataSync.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/discovery/TimelineMetricMetadataSync.java
deleted file mode 100644
index fa5f55a..0000000
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/discovery/TimelineMetricMetadataSync.java
+++ /dev/null
@@ -1,227 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p/>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p/>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.discovery;
-
-import java.sql.SQLException;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.metrics2.sink.timeline.TimelineMetricMetadata;
-
-/**
- * Sync metadata info with the store
- */
-public class TimelineMetricMetadataSync implements Runnable {
- private static final Log LOG = LogFactory.getLog(TimelineMetricMetadataSync.class);
-
- private final TimelineMetricMetadataManager cacheManager;
-
- public TimelineMetricMetadataSync(TimelineMetricMetadataManager cacheManager) {
- this.cacheManager = cacheManager;
- }
-
- @Override
- public void run() {
- LOG.debug("Persisting metric metadata...");
- persistMetricMetadata();
- LOG.debug("Persisting hosted apps metadata...");
- persistHostAppsMetadata();
- LOG.debug("Persisting hosted instance metadata...");
- persistHostInstancesMetadata();
- if (cacheManager.isDistributedModeEnabled()) {
- LOG.debug("Refreshing metric metadata...");
- refreshMetricMetadata();
- LOG.debug("Refreshing hosted apps metadata...");
- refreshHostAppsMetadata();
- LOG.debug("Refreshing hosted instances metadata...");
- refreshHostedInstancesMetadata();
- }
- }
-
- /**
- * Find metrics not persisted to store and persist them
- */
- private void persistMetricMetadata() {
- List<TimelineMetricMetadata> metadataToPersist = new ArrayList<>();
- // Find all entries to persist
- for (TimelineMetricMetadata metadata : cacheManager.getMetadataCache().values()) {
- if (!metadata.isPersisted()) {
- metadataToPersist.add(metadata);
- }
- }
- boolean markSuccess = false;
- if (!metadataToPersist.isEmpty()) {
- try {
- cacheManager.persistMetadata(metadataToPersist);
- markSuccess = true;
- } catch (SQLException e) {
- LOG.warn("Error persisting metadata.", e);
- }
- }
- // Mark corresponding entries as persisted to skip on next run
- if (markSuccess) {
- for (TimelineMetricMetadata metadata : metadataToPersist) {
- TimelineMetricMetadataKey key = new TimelineMetricMetadataKey(
- metadata.getMetricName(), metadata.getAppId(), metadata.getInstanceId()
- );
-
- // Mark entry as being persisted
- metadata.setIsPersisted(true);
- // Update cache
- cacheManager.getMetadataCache().put(key, metadata);
- }
- }
- }
-
- /**
- * Read all metric metadata and update cached values - HA mode
- */
- private void refreshMetricMetadata() {
- Map<TimelineMetricMetadataKey, TimelineMetricMetadata> metadataFromStore = null;
- try {
- metadataFromStore = cacheManager.getMetadataFromStore();
- } catch (SQLException e) {
- LOG.warn("Error refreshing metadata from store.", e);
- }
- if (metadataFromStore != null) {
- Map<TimelineMetricMetadataKey, TimelineMetricMetadata> cachedMetadata =
- cacheManager.getMetadataCache();
-
- for (Map.Entry<TimelineMetricMetadataKey, TimelineMetricMetadata> metadataEntry : metadataFromStore.entrySet()) {
- if (!cachedMetadata.containsKey(metadataEntry.getKey())) {
- cachedMetadata.put(metadataEntry.getKey(), metadataEntry.getValue());
- }
- }
- }
- }
-
- /**
- * Sync hosted apps data if needed
- */
- private void persistHostAppsMetadata() {
- if (cacheManager.syncHostedAppsMetadata()) {
- Map<String, TimelineMetricHostMetadata> persistedData = null;
- try {
- persistedData = cacheManager.getHostedAppsFromStore();
- } catch (SQLException e) {
- LOG.warn("Failed on fetching hosted apps data from store.", e);
- return; // Something wrong with store
- }
-
- Map<String, TimelineMetricHostMetadata> cachedData = cacheManager.getHostedAppsCache();
- Map<String, TimelineMetricHostMetadata> dataToSync = new HashMap<>();
- if (cachedData != null && !cachedData.isEmpty()) {
- for (Map.Entry<String, TimelineMetricHostMetadata> cacheEntry : cachedData.entrySet()) {
- // No persistence / stale data in store
- if (persistedData == null || persistedData.isEmpty() ||
- !persistedData.containsKey(cacheEntry.getKey()) ||
- !persistedData.get(cacheEntry.getKey()).getHostedApps().keySet().containsAll(cacheEntry.getValue().getHostedApps().keySet())) {
- dataToSync.put(cacheEntry.getKey(), cacheEntry.getValue());
- }
- }
- try {
- cacheManager.persistHostedAppsMetadata(dataToSync);
- cacheManager.markSuccessOnSyncHostedAppsMetadata();
-
- } catch (SQLException e) {
- LOG.warn("Error persisting hosted apps metadata.", e);
- }
- }
-
- }
- }
-
- /**
- * Sync apps instances data if needed
- */
- private void persistHostInstancesMetadata() {
- if (cacheManager.syncHostedInstanceMetadata()) {
- Map<String, Set<String>> persistedData = null;
- try {
- persistedData = cacheManager.getHostedInstancesFromStore();
- } catch (SQLException e) {
- LOG.warn("Failed on fetching hosted instances data from store.", e);
- return; // Something wrong with store
- }
-
- Map<String, Set<String>> cachedData = cacheManager.getHostedInstanceCache();
- Map<String, Set<String>> dataToSync = new HashMap<>();
- if (cachedData != null && !cachedData.isEmpty()) {
- for (Map.Entry<String, Set<String>> cacheEntry : cachedData.entrySet()) {
- // No persistence / stale data in store
- if (persistedData == null || persistedData.isEmpty() ||
- !persistedData.containsKey(cacheEntry.getKey()) ||
- !persistedData.get(cacheEntry.getKey()).containsAll(cacheEntry.getValue())) {
- dataToSync.put(cacheEntry.getKey(), cacheEntry.getValue());
- }
- }
- try {
- cacheManager.persistHostedInstanceMetadata(dataToSync);
- cacheManager.markSuccessOnSyncHostedInstanceMetadata();
-
- } catch (SQLException e) {
- LOG.warn("Error persisting hosted apps metadata.", e);
- }
- }
-
- }
- }
- /**
- * Read all hosted apps metadata and update cached values - HA
- */
- private void refreshHostAppsMetadata() {
- Map<String, TimelineMetricHostMetadata> hostedAppsDataFromStore = null;
- try {
- hostedAppsDataFromStore = cacheManager.getHostedAppsFromStore();
- } catch (SQLException e) {
- LOG.warn("Error refreshing metadata from store.", e);
- }
- if (hostedAppsDataFromStore != null) {
- Map<String, TimelineMetricHostMetadata> cachedData = cacheManager.getHostedAppsCache();
-
- for (Map.Entry<String, TimelineMetricHostMetadata> storeEntry : hostedAppsDataFromStore.entrySet()) {
- if (!cachedData.containsKey(storeEntry.getKey())) {
- cachedData.put(storeEntry.getKey(), storeEntry.getValue());
- }
- }
- }
- }
-
- private void refreshHostedInstancesMetadata() {
- Map<String, Set<String>> hostedInstancesFromStore = null;
- try {
- hostedInstancesFromStore = cacheManager.getHostedInstancesFromStore();
- } catch (SQLException e) {
- LOG.warn("Error refreshing metadata from store.", e);
- }
- if (hostedInstancesFromStore != null) {
- Map<String, Set<String>> cachedData = cacheManager.getHostedInstanceCache();
-
- for (Map.Entry<String, Set<String>> storeEntry : hostedInstancesFromStore.entrySet()) {
- if (!cachedData.containsKey(storeEntry.getKey())) {
- cachedData.put(storeEntry.getKey(), storeEntry.getValue());
- }
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/function/AbstractTimelineMetricsSeriesAggregateFunction.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/function/AbstractTimelineMetricsSeriesAggregateFunction.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/function/AbstractTimelineMetricsSeriesAggregateFunction.java
deleted file mode 100644
index 5a5dde4..0000000
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/function/AbstractTimelineMetricsSeriesAggregateFunction.java
+++ /dev/null
@@ -1,98 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.function;
-
-import java.util.Iterator;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.TreeMap;
-import java.util.TreeSet;
-
-import org.apache.hadoop.metrics2.sink.timeline.TimelineMetric;
-import org.apache.hadoop.metrics2.sink.timeline.TimelineMetrics;
-
-import com.google.common.base.Joiner;
-
-public abstract class AbstractTimelineMetricsSeriesAggregateFunction
- implements TimelineMetricsSeriesAggregateFunction {
-
- @Override
- public TimelineMetric apply(TimelineMetrics timelineMetrics) {
- Set<String> metricNameSet = new TreeSet<>();
- Set<String> hostNameSet = new TreeSet<>();
- Set<String> appIdSet = new TreeSet<>();
- Set<String> instanceIdSet = new TreeSet<>();
- TreeMap<Long, List<Double>> metricValues = new TreeMap<>();
-
- for (TimelineMetric timelineMetric : timelineMetrics.getMetrics()) {
- metricNameSet.add(timelineMetric.getMetricName());
- addToSetOnlyNotNull(hostNameSet, timelineMetric.getHostName());
- addToSetOnlyNotNull(appIdSet, timelineMetric.getAppId());
- addToSetOnlyNotNull(instanceIdSet, timelineMetric.getInstanceId());
-
- for (Map.Entry<Long, Double> metricValue : timelineMetric.getMetricValues().entrySet()) {
- Long timestamp = metricValue.getKey();
- Double value = metricValue.getValue();
- if (!metricValues.containsKey(timestamp)) {
- metricValues.put(timestamp, new LinkedList<Double>());
- }
- metricValues.get(timestamp).add(value);
- }
- }
-
- TreeMap<Long, Double> aggregatedMetricValues = new TreeMap<>();
- for (Map.Entry<Long, List<Double>> metricValue : metricValues.entrySet()) {
- List<Double> values = metricValue.getValue();
- if (values.size() == 0) {
- throw new IllegalArgumentException("count of values should be more than 0");
- }
- aggregatedMetricValues.put(metricValue.getKey(), applyFunction(values));
- }
-
- TimelineMetric timelineMetric = new TimelineMetric();
- timelineMetric.setMetricName(getMetricName(metricNameSet.iterator()));
- timelineMetric.setHostName(joinStringsWithComma(hostNameSet.iterator()));
- timelineMetric.setAppId(joinStringsWithComma(appIdSet.iterator()));
- timelineMetric.setInstanceId(joinStringsWithComma(instanceIdSet.iterator()));
- if (aggregatedMetricValues.size() > 0) {
- timelineMetric.setStartTime(aggregatedMetricValues.firstKey());
- }
- timelineMetric.setMetricValues(aggregatedMetricValues);
- return timelineMetric;
- }
-
- protected String getMetricName(Iterator<String> metricNames) {
- return getFunctionName() + "(" + Joiner.on(",").join(metricNames) + ")";
- }
-
- protected String joinStringsWithComma(Iterator<String> hostNames) {
- return Joiner.on(",").join(hostNames);
- }
-
- protected abstract Double applyFunction(List<Double> values);
- protected abstract String getFunctionName();
-
- private void addToSetOnlyNotNull(Set<String> set, String value) {
- if (value != null) {
- set.add(value);
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/function/SeriesAggregateFunction.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/function/SeriesAggregateFunction.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/function/SeriesAggregateFunction.java
deleted file mode 100644
index ef5e441..0000000
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/function/SeriesAggregateFunction.java
+++ /dev/null
@@ -1,42 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.function;
-
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.aggregators.Function;
-
-public enum SeriesAggregateFunction {
- AVG, MIN, MAX, SUM;
-
- public static boolean isPresent(String functionName) {
- try {
- SeriesAggregateFunction.valueOf(functionName.toUpperCase());
- } catch (IllegalArgumentException e) {
- return false;
- }
- return true;
- }
-
- public static SeriesAggregateFunction getFunction(String functionName) throws Function.FunctionFormatException {
- try {
- return SeriesAggregateFunction.valueOf(functionName.toUpperCase());
- } catch (NullPointerException | IllegalArgumentException e) {
- throw new Function.FunctionFormatException(
- "Function should be sum, avg, min, max. Got " + functionName, e);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/function/TimelineMetricsSeriesAggregateFunction.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/function/TimelineMetricsSeriesAggregateFunction.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/function/TimelineMetricsSeriesAggregateFunction.java
deleted file mode 100644
index bdb5fe5..0000000
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/function/TimelineMetricsSeriesAggregateFunction.java
+++ /dev/null
@@ -1,25 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.function;
-
-import org.apache.hadoop.metrics2.sink.timeline.TimelineMetric;
-import org.apache.hadoop.metrics2.sink.timeline.TimelineMetrics;
-
-public interface TimelineMetricsSeriesAggregateFunction {
- TimelineMetric apply(TimelineMetrics timelineMetrics);
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/function/TimelineMetricsSeriesAggregateFunctionFactory.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/function/TimelineMetricsSeriesAggregateFunctionFactory.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/function/TimelineMetricsSeriesAggregateFunctionFactory.java
deleted file mode 100644
index 63a0fdc..0000000
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/function/TimelineMetricsSeriesAggregateFunctionFactory.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.function;
-
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.aggregators.Function;
-
-public class TimelineMetricsSeriesAggregateFunctionFactory {
- private TimelineMetricsSeriesAggregateFunctionFactory() {
- }
-
- public static TimelineMetricsSeriesAggregateFunction newInstance(SeriesAggregateFunction func) {
- switch (func) {
- case AVG:
- return new TimelineMetricsSeriesAvgAggregateFunction();
- case MIN:
- return new TimelineMetricsSeriesMinAggregateFunction();
- case MAX:
- return new TimelineMetricsSeriesMaxAggregateFunction();
- case SUM:
- return new TimelineMetricsSeriesSumAggregateFunction();
- default:
- throw new Function.FunctionFormatException("Function should be sum, avg, min, max. Got " +
- func.name());
- }
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/function/TimelineMetricsSeriesAvgAggregateFunction.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/function/TimelineMetricsSeriesAvgAggregateFunction.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/function/TimelineMetricsSeriesAvgAggregateFunction.java
deleted file mode 100644
index f7c66ed..0000000
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/function/TimelineMetricsSeriesAvgAggregateFunction.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.function;
-
-import java.util.List;
-
-public class TimelineMetricsSeriesAvgAggregateFunction extends AbstractTimelineMetricsSeriesAggregateFunction {
- private static final String FUNCTION_NAME = "AVG";
-
- @Override
- protected Double applyFunction(List<Double> values) {
- double sum = 0.0d;
- for (Double value : values) {
- sum += value;
- }
-
- return sum / values.size();
- }
-
- @Override
- protected String getFunctionName() {
- return FUNCTION_NAME;
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/function/TimelineMetricsSeriesMaxAggregateFunction.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/function/TimelineMetricsSeriesMaxAggregateFunction.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/function/TimelineMetricsSeriesMaxAggregateFunction.java
deleted file mode 100644
index 0b79f78..0000000
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/function/TimelineMetricsSeriesMaxAggregateFunction.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.function;
-
-import java.util.List;
-
-public class TimelineMetricsSeriesMaxAggregateFunction extends AbstractTimelineMetricsSeriesAggregateFunction {
- private static final String FUNCTION_NAME = "MAX";
-
- @Override
- protected Double applyFunction(List<Double> values) {
- double max = Double.MIN_VALUE;
- for (Double value : values) {
- if (value > max) {
- max = value;
- }
- }
-
- return max;
- }
-
- @Override
- protected String getFunctionName() {
- return FUNCTION_NAME;
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/function/TimelineMetricsSeriesMinAggregateFunction.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/function/TimelineMetricsSeriesMinAggregateFunction.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/function/TimelineMetricsSeriesMinAggregateFunction.java
deleted file mode 100644
index 7146aa2..0000000
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/function/TimelineMetricsSeriesMinAggregateFunction.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.function;
-
-import java.util.List;
-
-public class TimelineMetricsSeriesMinAggregateFunction extends AbstractTimelineMetricsSeriesAggregateFunction {
- private static final String FUNCTION_NAME = "MIN";
-
- @Override
- protected Double applyFunction(List<Double> values) {
- double min = Double.MAX_VALUE;
- for (Double value : values) {
- if (value < min) {
- min = value;
- }
- }
-
- return min;
- }
-
- @Override
- protected String getFunctionName() {
- return FUNCTION_NAME;
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/function/TimelineMetricsSeriesSumAggregateFunction.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/function/TimelineMetricsSeriesSumAggregateFunction.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/function/TimelineMetricsSeriesSumAggregateFunction.java
deleted file mode 100644
index 2a15c95..0000000
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/function/TimelineMetricsSeriesSumAggregateFunction.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.function;
-
-import java.util.List;
-
-public class TimelineMetricsSeriesSumAggregateFunction extends AbstractTimelineMetricsSeriesAggregateFunction {
- private static final String FUNCTION_NAME = "SUM";
-
- @Override
- protected Double applyFunction(List<Double> values) {
- double sum = 0.0d;
- for (Double value : values) {
- sum += value;
- }
-
- return sum;
- }
-
- @Override
- protected String getFunctionName() {
- return FUNCTION_NAME;
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/query/Condition.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/query/Condition.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/query/Condition.java
deleted file mode 100644
index 8d8cca3..0000000
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/query/Condition.java
+++ /dev/null
@@ -1,51 +0,0 @@
-package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query;
-
-import java.util.List;
-
-import org.apache.hadoop.metrics2.sink.timeline.Precision;
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p/>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p/>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-public interface Condition {
- boolean isEmpty();
-
- List<byte[]> getUuids();
- List<String> getMetricNames();
- boolean isPointInTime();
- boolean isGrouped();
- void setStatement(String statement);
- List<String> getHostnames();
- Precision getPrecision();
- void setPrecision(Precision precision);
- String getAppId();
- String getInstanceId();
- StringBuilder getConditionClause();
- String getOrderByClause(boolean asc);
- String getStatement();
- Long getStartTime();
- Long getEndTime();
- Integer getLimit();
- Integer getFetchSize();
- void setFetchSize(Integer fetchSize);
- void addOrderByColumn(String column);
- void setNoLimit();
- boolean doUpdate();
- void setMetricNamesNotCondition(boolean metricNamesNotCondition);
- void setHostnamesNotCondition(boolean hostNamesNotCondition);
- void setUuidNotCondition(boolean uuidNotCondition);
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/query/ConditionBuilder.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/query/ConditionBuilder.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/query/ConditionBuilder.java
deleted file mode 100644
index f330b60..0000000
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/query/ConditionBuilder.java
+++ /dev/null
@@ -1,143 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query;
-
-import java.util.LinkedHashSet;
-import java.util.List;
-import java.util.Set;
-
-import org.apache.hadoop.metrics2.sink.timeline.Precision;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.aggregators.Function;
-
-public class ConditionBuilder {
-
- private List<String> metricNames;
- private List<String> hostnames;
- private String appId;
- private String instanceId;
- private Long startTime;
- private Long endTime;
- private Precision precision;
- private Integer limit;
- private boolean grouped;
- private boolean noLimit = false;
- private Integer fetchSize;
- private String statement;
- private Set<String> orderByColumns = new LinkedHashSet<String>();
- private Integer topN;
- private boolean isBottomN;
- private Function topNFunction;
- private List<byte[]> uuids;
-
- public ConditionBuilder(List<String> metricNames) {
- this.metricNames = metricNames;
- }
-
- public ConditionBuilder hostnames(List<String> hostnames) {
- this.hostnames = hostnames;
- return this;
- }
-
- public ConditionBuilder appId(String appId) {
- this.appId = appId;
- return this;
- }
-
- public ConditionBuilder instanceId(String instanceId) {
- this.instanceId = instanceId;
- return this;
- }
-
- public ConditionBuilder startTime(Long startTime) {
- this.startTime = startTime;
- return this;
- }
-
- public ConditionBuilder endTime(Long endTime) {
- this.endTime = endTime;
- return this;
- }
-
- public ConditionBuilder precision(Precision precision) {
- this.precision = precision;
- return this;
- }
-
- public ConditionBuilder limit(Integer limit) {
- this.limit = limit;
- return this;
- }
-
- public ConditionBuilder grouped(boolean grouped) {
- this.grouped = grouped;
- return this;
- }
-
- public ConditionBuilder noLimit(boolean noLimit) {
- this.noLimit = noLimit;
- return this;
- }
-
- public ConditionBuilder fetchSize(Integer fetchSize) {
- this.fetchSize = fetchSize;
- return this;
- }
-
- public ConditionBuilder statement(String statement) {
- this.statement = statement;
- return this;
- }
-
- public ConditionBuilder orderByColumns(Set<String> orderByColumns) {
- this.orderByColumns = orderByColumns;
- return this;
- }
-
- public ConditionBuilder topN(Integer topN) {
- this.topN = topN;
- return this;
- }
-
- public ConditionBuilder isBottomN(boolean isBottomN) {
- this.isBottomN = isBottomN;
- return this;
- }
-
- public ConditionBuilder topNFunction(Function topNFunction) {
- this.topNFunction = topNFunction;
- return this;
- }
-
- public ConditionBuilder uuid(List<byte[]> uuids) {
- this.uuids = uuids;
- return this;
- }
-
- public Condition build() {
- if (topN == null) {
- return new DefaultCondition(
- uuids, metricNames,
- hostnames, appId, instanceId, startTime, endTime,
- precision, limit, grouped);
- } else {
- return new TopNCondition(uuids, metricNames, hostnames, appId, instanceId,
- startTime, endTime, precision, limit, grouped, topN, topNFunction, isBottomN);
- }
- }
-
-}
[20/28] ambari git commit: AMBARI-22740 : Fix integration test for
HBase in branch-3.0-ams due to UUID changes. (avijayan)
Posted by av...@apache.org.
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/source/cache/InternalMetricsCache.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/source/cache/InternalMetricsCache.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/source/cache/InternalMetricsCache.java
new file mode 100644
index 0000000..b26921f
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/source/cache/InternalMetricsCache.java
@@ -0,0 +1,229 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.metrics.core.timeline.source.cache;
+
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Date;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReentrantLock;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.metrics2.sink.timeline.TimelineMetric;
+import org.apache.hadoop.metrics2.sink.timeline.TimelineMetrics;
+
+import net.sf.ehcache.Cache;
+import net.sf.ehcache.CacheException;
+import net.sf.ehcache.CacheManager;
+import net.sf.ehcache.Ehcache;
+import net.sf.ehcache.Element;
+import net.sf.ehcache.config.CacheConfiguration;
+import net.sf.ehcache.config.PersistenceConfiguration;
+import net.sf.ehcache.config.SizeOfPolicyConfiguration;
+import net.sf.ehcache.event.CacheEventListener;
+import net.sf.ehcache.store.MemoryStoreEvictionPolicy;
+
+public class InternalMetricsCache {
+ private static final Log LOG = LogFactory.getLog(InternalMetricsCache.class);
+ private final String instanceName;
+ private final String maxHeapPercent;
+ private volatile boolean isCacheInitialized = false;
+ private Cache cache;
+ static final String TIMELINE_METRIC_CACHE_MANAGER_NAME = "internalMetricsCacheManager";
+ private final Lock lock = new ReentrantLock();
+ private static final int LOCK_TIMEOUT_SECONDS = 2;
+
+ public InternalMetricsCache(String instanceName, String maxHeapPercent) {
+ this.instanceName = instanceName;
+ this.maxHeapPercent = maxHeapPercent;
+ initialize();
+ }
+
+ private void initialize() {
+ // Check in case of contention to avoid ObjectExistsException
+ if (isCacheInitialized) {
+ throw new RuntimeException("Cannot initialize internal cache twice");
+ }
+
+ System.setProperty("net.sf.ehcache.skipUpdateCheck", "true");
+ System.setProperty("net.sf.ehcache.sizeofengine." + TIMELINE_METRIC_CACHE_MANAGER_NAME,
+ "org.apache.ambari.metrics.core.timeline.source.cache.InternalMetricsCacheSizeOfEngine");
+
+ net.sf.ehcache.config.Configuration managerConfig =
+ new net.sf.ehcache.config.Configuration();
+ managerConfig.setName(TIMELINE_METRIC_CACHE_MANAGER_NAME);
+
+ // Set max heap available to the cache manager
+ managerConfig.setMaxBytesLocalHeap(maxHeapPercent);
+
+ //Create a singleton CacheManager using defaults
+ CacheManager manager = CacheManager.create(managerConfig);
+
+ LOG.info("Creating Metrics Cache with maxHeapPercent => " + maxHeapPercent);
+
+ // Create a Cache specifying its configuration.
+ CacheConfiguration cacheConfiguration = new CacheConfiguration()
+ .name(instanceName)
+ .memoryStoreEvictionPolicy(MemoryStoreEvictionPolicy.LRU)
+ .sizeOfPolicy(new SizeOfPolicyConfiguration() // Set sizeOf policy to continue on max depth reached - avoid OOM
+ .maxDepth(10000)
+ .maxDepthExceededBehavior(SizeOfPolicyConfiguration.MaxDepthExceededBehavior.CONTINUE))
+ .eternal(true) // infinite time until eviction
+ .persistence(new PersistenceConfiguration()
+ .strategy(PersistenceConfiguration.Strategy.NONE.name()));
+
+ cache = new Cache(cacheConfiguration);
+ cache.getCacheEventNotificationService().registerListener(new InternalCacheEvictionListener());
+
+ LOG.info("Registering internal metrics cache with provider: name = " +
+ cache.getName() + ", guid: " + cache.getGuid());
+
+ manager.addCache(cache);
+
+ isCacheInitialized = true;
+ }
+
+ public InternalMetricCacheValue getInternalMetricCacheValue(InternalMetricCacheKey key) {
+ Element ele = cache.get(key);
+ if (ele != null) {
+ return (InternalMetricCacheValue) ele.getObjectValue();
+ }
+ return null;
+ }
+
+ public Collection<TimelineMetrics> evictAll() {
+ TimelineMetrics metrics = new TimelineMetrics();
+ try {
+ if (lock.tryLock(LOCK_TIMEOUT_SECONDS, TimeUnit.SECONDS)) {
+ try{
+ List keys = cache.getKeys();
+ for (Object obj : keys) {
+ TimelineMetric metric = new TimelineMetric();
+ InternalMetricCacheKey key = (InternalMetricCacheKey) obj;
+ metric.setMetricName(key.getMetricName());
+ metric.setAppId(key.getAppId());
+ metric.setInstanceId(key.getInstanceId());
+ metric.setHostName(key.getHostname());
+ metric.setStartTime(key.getStartTime());
+ Element ele = cache.get(key);
+ metric.setMetricValues(((InternalMetricCacheValue) ele.getObjectValue()).getMetricValues());
+ metrics.getMetrics().add(metric);
+ }
+ cache.removeAll();
+ } finally {
+ lock.unlock();
+ }
+ } else {
+ LOG.warn("evictAll: Unable to acquire lock on the cache instance. " +
+ "Giving up after " + LOCK_TIMEOUT_SECONDS + " seconds.");
+ }
+ } catch (InterruptedException e) {
+ LOG.warn("Interrupted while waiting to acquire lock");
+ }
+
+ return Collections.singletonList(metrics);
+ }
+
+ public void putAll(Collection<TimelineMetrics> metrics) {
+ try {
+ if (lock.tryLock(LOCK_TIMEOUT_SECONDS, TimeUnit.SECONDS)) {
+ try {
+ if (metrics != null) {
+ for (TimelineMetrics timelineMetrics : metrics) {
+ for (TimelineMetric timelineMetric : timelineMetrics.getMetrics()) {
+ InternalMetricCacheKey key = new InternalMetricCacheKey(
+ timelineMetric.getMetricName(),
+ timelineMetric.getAppId(),
+ timelineMetric.getInstanceId(),
+ timelineMetric.getHostName(),
+ timelineMetric.getStartTime()
+ );
+
+ Element ele = cache.get(key);
+ if (ele != null) {
+ InternalMetricCacheValue value = (InternalMetricCacheValue) ele.getObjectValue();
+ value.addMetricValues(timelineMetric.getMetricValues());
+ } else {
+ InternalMetricCacheValue value = new InternalMetricCacheValue();
+ value.setMetricValues(timelineMetric.getMetricValues());
+ cache.put(new Element(key, value));
+ }
+ }
+ }
+ }
+ } finally {
+ lock.unlock();
+ }
+ } else {
+ LOG.warn("putAll: Unable to acquire lock on the cache instance. " +
+ "Giving up after " + LOCK_TIMEOUT_SECONDS + " seconds.");
+ }
+ } catch (InterruptedException e) {
+ LOG.warn("Interrupted while waiting to acquire lock");
+ }
+ }
+
+ class InternalCacheEvictionListener implements CacheEventListener {
+
+ @Override
+ public void notifyElementRemoved(Ehcache cache, Element element) throws CacheException {
+ // expected
+ }
+
+ @Override
+ public void notifyElementPut(Ehcache cache, Element element) throws CacheException {
+ // do nothing
+ }
+
+ @Override
+ public void notifyElementUpdated(Ehcache cache, Element element) throws CacheException {
+ // do nothing
+ }
+
+ @Override
+ public void notifyElementExpired(Ehcache cache, Element element) {
+ // do nothing
+ }
+
+ @Override
+ public void notifyElementEvicted(Ehcache cache, Element element) {
+ // Bad - Remote endpoint cannot keep up resulting in flooding
+ InternalMetricCacheKey key = (InternalMetricCacheKey) element.getObjectKey();
+ LOG.warn("Evicting element from internal metrics cache, metric => " + key
+ .getMetricName() + ", startTime = " + new Date(key.getStartTime()));
+ }
+
+ @Override
+ public void notifyRemoveAll(Ehcache cache) {
+ // expected
+ }
+
+ @Override
+ public Object clone() throws CloneNotSupportedException {
+ return null;
+ }
+
+ @Override
+ public void dispose() {
+ // do nothing
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/source/cache/InternalMetricsCacheProvider.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/source/cache/InternalMetricsCacheProvider.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/source/cache/InternalMetricsCacheProvider.java
new file mode 100644
index 0000000..8ec9acd
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/source/cache/InternalMetricsCacheProvider.java
@@ -0,0 +1,48 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.metrics.core.timeline.source.cache;
+
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+
+import org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguration;
+
+public class InternalMetricsCacheProvider {
+ private Map<String, InternalMetricsCache> metricsCacheMap = new ConcurrentHashMap<>();
+ private static final InternalMetricsCacheProvider instance = new InternalMetricsCacheProvider();
+
+ private InternalMetricsCacheProvider() {
+ }
+
+ public static InternalMetricsCacheProvider getInstance() {
+ return instance;
+ }
+
+ public InternalMetricsCache getCacheInstance(String instanceName) {
+ if (metricsCacheMap.containsKey(instanceName)) {
+ return metricsCacheMap.get(instanceName);
+ } else {
+ TimelineMetricConfiguration conf = TimelineMetricConfiguration.getInstance();
+ InternalMetricsCache cache = new InternalMetricsCache(instanceName,
+ conf.getInternalCacheHeapPercent(instanceName));
+
+ metricsCacheMap.put(instanceName, cache);
+ return cache;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/source/cache/InternalMetricsCacheSizeOfEngine.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/source/cache/InternalMetricsCacheSizeOfEngine.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/source/cache/InternalMetricsCacheSizeOfEngine.java
new file mode 100644
index 0000000..1c5b839
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/source/cache/InternalMetricsCacheSizeOfEngine.java
@@ -0,0 +1,52 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.metrics.core.timeline.source.cache;
+
+import org.apache.hadoop.metrics2.sink.timeline.cache.TimelineMetricsEhCacheSizeOfEngine;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class InternalMetricsCacheSizeOfEngine extends TimelineMetricsEhCacheSizeOfEngine {
+ private final static Logger LOG = LoggerFactory.getLogger(InternalMetricsCacheSizeOfEngine.class);
+
+ public InternalMetricsCacheSizeOfEngine() {
+ // Invoke default constructor in base class
+ }
+
+ @Override
+ protected long getSizeOfEntry(Object key, Object value) {
+ try {
+ LOG.debug("BEGIN - Sizeof, key: {}, value: {}", key, value);
+ long size = 0;
+ if (key instanceof InternalMetricCacheKey) {
+ InternalMetricCacheKey metricCacheKey = (InternalMetricCacheKey) key;
+ size += reflectionSizeOf.sizeOf(metricCacheKey.getMetricName());
+ size += reflectionSizeOf.sizeOf(metricCacheKey.getAppId());
+ size += reflectionSizeOf.sizeOf(metricCacheKey.getInstanceId()); // null safe
+ size += reflectionSizeOf.sizeOf(metricCacheKey.getHostname());
+ }
+ if (value instanceof InternalMetricCacheValue) {
+ size += getValueMapSize(((InternalMetricCacheValue) value).getMetricValues());
+ }
+ // Mark size as not being exact
+ return size;
+ } finally {
+ LOG.debug("END - Sizeof, key: {}", key);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/uuid/HashBasedUuidGenStrategy.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/uuid/HashBasedUuidGenStrategy.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/uuid/HashBasedUuidGenStrategy.java
new file mode 100644
index 0000000..203c3a0
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/uuid/HashBasedUuidGenStrategy.java
@@ -0,0 +1,209 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.metrics.core.timeline.uuid;
+
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+import org.apache.ambari.metrics.core.timeline.aggregators.TimelineClusterMetric;
+import org.apache.commons.lang.ArrayUtils;
+import org.apache.commons.lang3.StringUtils;
+
+public class HashBasedUuidGenStrategy implements MetricUuidGenStrategy {
+
+ /**
+ * Computes the UUID for a timelineClusterMetric.
+ * @param timelineClusterMetric
+ * @param maxLength
+ * @return byte array of length 'maxlength'
+ */
+ @Override
+ public byte[] computeUuid(TimelineClusterMetric timelineClusterMetric, int maxLength) {
+
+ int metricNameUuidLength = 12;
+ String metricName = timelineClusterMetric.getMetricName();
+
+ //Compute the individual splits.
+ String[] splits = getIndidivualSplits(metricName);
+
+ /*
+ Compute the ascii sum of every split in the metric name. (asciiSum += (int) splits[s].charAt(i))
+ For the last split, use weighted sum instead of ascii sum. (asciiSum += ((i+1) * (int) splits[s].charAt(i)))
+ These weighted sums are 'appended' to get the unique ID for metric name.
+ */
+ StringBuilder splitSums = new StringBuilder();
+ if (splits.length > 0) {
+ for (int s = 0; s < splits.length; s++) {
+ int asciiSum = 0;
+ if ( s < splits.length -1) {
+ for (int i = 0; i < splits[s].length(); i++) {
+ asciiSum += (int) splits[s].charAt(i); // Get Ascii Sum.
+ }
+ } else {
+ for (int i = 0; i < splits[s].length(); i++) {
+ asciiSum += ((i+1) * (int) splits[s].charAt(i)); //weighted sum for last split.
+ }
+ }
+ splitSums.append(asciiSum); //Append the sum to the array of sums.
+ }
+ }
+
+ //Compute a unique metric seed for the stemmed metric name
+ String stemmedMetric = stem(metricName);
+ long metricSeed = 100123456789L;
+ for (int i = 0; i < stemmedMetric.length(); i++) {
+ metricSeed += stemmedMetric.charAt(i);
+ }
+
+ //Reverse the computed seed to get a metric UUID portion which is used optionally.
+ byte[] metricUuidPortion = StringUtils.reverse(String.valueOf(metricSeed)).getBytes();
+ String splitSumString = splitSums.toString();
+ int splitLength = splitSumString.length();
+
+ //If splitSums length > required metric UUID length, use only the required length suffix substring of the splitSums as metric UUID.
+ if (splitLength > metricNameUuidLength) {
+ metricUuidPortion = ArrayUtils.subarray(splitSumString.getBytes(), splitLength - metricNameUuidLength, splitLength);
+ } else {
+ //If splitSums is not enough for required metric UUID length, pad with the metric uuid portion.
+ int pad = metricNameUuidLength - splitLength;
+ metricUuidPortion = ArrayUtils.addAll(splitSumString.getBytes(), ArrayUtils.subarray(metricUuidPortion, 0, pad));
+ }
+
+ /*
+ For appId and instanceId the logic is similar. Use a seed integer to start with and compute ascii sum.
+ Based on required length, use a suffix of the computed uuid.
+ */
+ byte[] appUuidPortion = new byte[2];
+ String appId = timelineClusterMetric.getAppId();
+ if (StringUtils.isNotEmpty(appId)) {
+ int appidSeed = 11;
+ for (int i = 0; i < appId.length(); i++) {
+ appidSeed += appId.charAt(i);
+ }
+ String appIdSeedStr = String.valueOf(appidSeed);
+ appUuidPortion = ArrayUtils.subarray(appIdSeedStr.getBytes(), appIdSeedStr.length() - 2, appIdSeedStr.length());
+ }
+
+ String instanceId = timelineClusterMetric.getInstanceId();
+ ByteBuffer buffer = ByteBuffer.allocate(4);
+ byte[] instanceUuidPortion = new byte[2];
+ if (StringUtils.isNotEmpty(instanceId)) {
+ int instanceIdSeed = 1489;
+ for (int i = 0; i < appId.length(); i++) {
+ instanceIdSeed += appId.charAt(i);
+ }
+ buffer.putInt(instanceIdSeed);
+ ArrayUtils.subarray(buffer.array(), 2, 4);
+ }
+
+ // Concatenate all UUIDs together (metric uuid + appId uuid + instanceId uuid)
+ return ArrayUtils.addAll(ArrayUtils.addAll(metricUuidPortion, appUuidPortion), instanceUuidPortion);
+ }
+
+ /**
+ * Splits the metric name into individual tokens.
+ * For example,
+ * kafka.server.ReplicaManager.LeaderCount -> [kafka, server, ReplicaManager, LeaderCount]
+ * default.General.api_drop_table_15min_rate -> [default, General, api, drop, table, 15min, rate]
+ * @param metricName
+ * @return
+ */
+ private String[] getIndidivualSplits(String metricName) {
+ List<String> tokens = new ArrayList<>();
+ String[] splits = new String[0];
+ if (metricName.contains("\\.")) {
+ splits = metricName.split("\\.");
+ for (String split : splits) {
+ if (split.contains("_")) {
+ tokens.addAll(Arrays.asList(split.split("_")));
+ } else {
+ tokens.add(split);
+ }
+ }
+ }
+
+ if (splits.length <= 1) {
+ splits = metricName.split("\\_");
+ return splits;
+ }
+
+ if (splits.length <= 1) {
+ splits = metricName.split("\\=");
+ return splits;
+ }
+
+ return tokens.toArray(new String[tokens.size()]);
+ }
+
+ /**
+ * Stem the metric name. Remove a set of usual suspects characters.
+ * @param metricName
+ * @return
+ */
+ private String stem(String metricName) {
+ String metric = metricName.toLowerCase();
+ String regex = "[\\.\\_\\%\\-\\=]";
+ String trimmedMetric = StringUtils.removePattern(metric, regex);
+ return trimmedMetric;
+ }
+
+
+ /**
+ * Computes the UUID of a string. (hostname)
+ * Uses the ascii sum of the String. Numbers in the String are treated as actual numerical values rather than ascii values.
+ * @param value
+ * @param maxLength
+ * @return byte array of length 'maxlength'
+ */
+ @Override
+ public byte[] computeUuid(String value, int maxLength) {
+
+ if (StringUtils.isEmpty(value)) {
+ return null;
+ }
+ int len = value.length();
+ int numericValue = 0;
+ int seed = 1489;
+ for (int i = 0; i < len; i++) {
+ int ascii = value.charAt(i);
+ if (48 <= ascii && ascii <= 57) {
+ numericValue += numericValue * 10 + (ascii - 48);
+ } else {
+ if (numericValue > 0) {
+ seed += numericValue;
+ numericValue = 0;
+ }
+ seed+= value.charAt(i);
+ }
+ }
+
+ if (numericValue != 0) {
+ seed+=numericValue;
+ }
+
+ String seedStr = String.valueOf(seed);
+ if (seedStr.length() < maxLength) {
+ return null;
+ } else {
+ return seedStr.substring(seedStr.length() - maxLength, seedStr.length()).getBytes();
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/uuid/MetricUuidGenStrategy.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/uuid/MetricUuidGenStrategy.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/uuid/MetricUuidGenStrategy.java
new file mode 100644
index 0000000..5464d10
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/uuid/MetricUuidGenStrategy.java
@@ -0,0 +1,48 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.metrics.core.timeline.uuid;
+
+import org.apache.ambari.metrics.core.timeline.aggregators.TimelineClusterMetric;
+
+public interface MetricUuidGenStrategy {
+
+ /**
+ * Compute UUID for a given value
+ * @param timelineMetric instance
+ * @param maxLength
+ * @return
+ */
+// byte[] computeUuid(TimelineMetric timelineMetric, int maxLength);
+
+ /**
+ * Compute UUID for a given value
+ * @param value
+ * @param maxLength
+ * @return
+ */
+ byte[] computeUuid(TimelineClusterMetric timelineClusterMetric, int maxLength);
+
+ /**
+ * Compute UUID for a given value
+ * @param value
+ * @param maxLength
+ * @return
+ */
+ byte[] computeUuid(String value, int maxLength);
+
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/uuid/RandomUuidGenStrategy.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/uuid/RandomUuidGenStrategy.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/uuid/RandomUuidGenStrategy.java
new file mode 100644
index 0000000..613b566
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/uuid/RandomUuidGenStrategy.java
@@ -0,0 +1,53 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.metrics.core.timeline.uuid;
+
+import java.security.SecureRandom;
+
+import org.apache.ambari.metrics.core.timeline.aggregators.TimelineClusterMetric;
+
+import com.google.common.primitives.Longs;
+
+public class RandomUuidGenStrategy implements MetricUuidGenStrategy {
+ private static SecureRandom randomGenerator;
+
+ public RandomUuidGenStrategy() {
+ randomGenerator = new SecureRandom(
+ Longs.toByteArray(System.currentTimeMillis()));
+ }
+
+ @Override
+ public byte[] computeUuid(TimelineClusterMetric timelineClusterMetric, int maxLength) {
+ final byte[] bytes = new byte[maxLength];
+ randomGenerator.nextBytes(bytes);
+ return bytes;
+ }
+
+// @Override
+// public byte[] computeUuid(TimelineMetric timelineMetric, int maxLength) {
+// return new byte[10];
+// }
+
+ @Override
+ public byte[] computeUuid(String value, int maxLength) {
+ final byte[] bytes = new byte[maxLength];
+ randomGenerator.nextBytes(bytes);
+ return bytes;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/records/ApplicationAttemptFinishData.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/records/ApplicationAttemptFinishData.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/records/ApplicationAttemptFinishData.java
new file mode 100644
index 0000000..43e0e3b
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/records/ApplicationAttemptFinishData.java
@@ -0,0 +1,95 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.metrics.records;
+
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
+import org.apache.hadoop.yarn.api.records.YarnApplicationAttemptState;
+import org.apache.hadoop.yarn.util.Records;
+
+/**
+ * The class contains the fields that can be determined when
+ * <code>RMAppAttempt</code> finishes, and that need to be stored persistently.
+ */
+@Public
+@Unstable
+public abstract class ApplicationAttemptFinishData {
+
+ @Public
+ @Unstable
+ public static ApplicationAttemptFinishData newInstance(
+ ApplicationAttemptId appAttemptId, String diagnosticsInfo,
+ String trackingURL, FinalApplicationStatus finalApplicationStatus,
+ YarnApplicationAttemptState yarnApplicationAttemptState) {
+ ApplicationAttemptFinishData appAttemptFD =
+ Records.newRecord(ApplicationAttemptFinishData.class);
+ appAttemptFD.setApplicationAttemptId(appAttemptId);
+ appAttemptFD.setDiagnosticsInfo(diagnosticsInfo);
+ appAttemptFD.setTrackingURL(trackingURL);
+ appAttemptFD.setFinalApplicationStatus(finalApplicationStatus);
+ appAttemptFD.setYarnApplicationAttemptState(yarnApplicationAttemptState);
+ return appAttemptFD;
+ }
+
+ @Public
+ @Unstable
+ public abstract ApplicationAttemptId getApplicationAttemptId();
+
+ @Public
+ @Unstable
+ public abstract void setApplicationAttemptId(
+ ApplicationAttemptId applicationAttemptId);
+
+ @Public
+ @Unstable
+ public abstract String getTrackingURL();
+
+ @Public
+ @Unstable
+ public abstract void setTrackingURL(String trackingURL);
+
+ @Public
+ @Unstable
+ public abstract String getDiagnosticsInfo();
+
+ @Public
+ @Unstable
+ public abstract void setDiagnosticsInfo(String diagnosticsInfo);
+
+ @Public
+ @Unstable
+ public abstract FinalApplicationStatus getFinalApplicationStatus();
+
+ @Public
+ @Unstable
+ public abstract void setFinalApplicationStatus(
+ FinalApplicationStatus finalApplicationStatus);
+
+ @Public
+ @Unstable
+ public abstract YarnApplicationAttemptState getYarnApplicationAttemptState();
+
+ @Public
+ @Unstable
+ public abstract void setYarnApplicationAttemptState(
+ YarnApplicationAttemptState yarnApplicationAttemptState);
+
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/records/ApplicationAttemptHistoryData.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/records/ApplicationAttemptHistoryData.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/records/ApplicationAttemptHistoryData.java
new file mode 100644
index 0000000..a26e21a
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/records/ApplicationAttemptHistoryData.java
@@ -0,0 +1,171 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.metrics.records;
+
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
+import org.apache.hadoop.yarn.api.records.YarnApplicationAttemptState;
+
+/**
+ * The class contains all the fields that are stored persistently for
+ * <code>RMAppAttempt</code>.
+ */
+@Public
+@Unstable
+public class ApplicationAttemptHistoryData {
+
+ private ApplicationAttemptId applicationAttemptId;
+
+ private String host;
+
+ private int rpcPort;
+
+ private String trackingURL;
+
+ private String diagnosticsInfo;
+
+ private FinalApplicationStatus finalApplicationStatus;
+
+ private ContainerId masterContainerId;
+
+ private YarnApplicationAttemptState yarnApplicationAttemptState;
+
+ @Public
+ @Unstable
+ public static ApplicationAttemptHistoryData newInstance(
+ ApplicationAttemptId appAttemptId, String host, int rpcPort,
+ ContainerId masterContainerId, String diagnosticsInfo,
+ String trackingURL, FinalApplicationStatus finalApplicationStatus,
+ YarnApplicationAttemptState yarnApplicationAttemptState) {
+ ApplicationAttemptHistoryData appAttemptHD =
+ new ApplicationAttemptHistoryData();
+ appAttemptHD.setApplicationAttemptId(appAttemptId);
+ appAttemptHD.setHost(host);
+ appAttemptHD.setRPCPort(rpcPort);
+ appAttemptHD.setMasterContainerId(masterContainerId);
+ appAttemptHD.setDiagnosticsInfo(diagnosticsInfo);
+ appAttemptHD.setTrackingURL(trackingURL);
+ appAttemptHD.setFinalApplicationStatus(finalApplicationStatus);
+ appAttemptHD.setYarnApplicationAttemptState(yarnApplicationAttemptState);
+ return appAttemptHD;
+ }
+
+ @Public
+ @Unstable
+ public ApplicationAttemptId getApplicationAttemptId() {
+ return applicationAttemptId;
+ }
+
+ @Public
+ @Unstable
+ public void
+ setApplicationAttemptId(ApplicationAttemptId applicationAttemptId) {
+ this.applicationAttemptId = applicationAttemptId;
+ }
+
+ @Public
+ @Unstable
+ public String getHost() {
+ return host;
+ }
+
+ @Public
+ @Unstable
+ public void setHost(String host) {
+ this.host = host;
+ }
+
+ @Public
+ @Unstable
+ public int getRPCPort() {
+ return rpcPort;
+ }
+
+ @Public
+ @Unstable
+ public void setRPCPort(int rpcPort) {
+ this.rpcPort = rpcPort;
+ }
+
+ @Public
+ @Unstable
+ public String getTrackingURL() {
+ return trackingURL;
+ }
+
+ @Public
+ @Unstable
+ public void setTrackingURL(String trackingURL) {
+ this.trackingURL = trackingURL;
+ }
+
+ @Public
+ @Unstable
+ public String getDiagnosticsInfo() {
+ return diagnosticsInfo;
+ }
+
+ @Public
+ @Unstable
+ public void setDiagnosticsInfo(String diagnosticsInfo) {
+ this.diagnosticsInfo = diagnosticsInfo;
+ }
+
+ @Public
+ @Unstable
+ public FinalApplicationStatus getFinalApplicationStatus() {
+ return finalApplicationStatus;
+ }
+
+ @Public
+ @Unstable
+ public void setFinalApplicationStatus(
+ FinalApplicationStatus finalApplicationStatus) {
+ this.finalApplicationStatus = finalApplicationStatus;
+ }
+
+ @Public
+ @Unstable
+ public ContainerId getMasterContainerId() {
+ return masterContainerId;
+ }
+
+ @Public
+ @Unstable
+ public void setMasterContainerId(ContainerId masterContainerId) {
+ this.masterContainerId = masterContainerId;
+ }
+
+ @Public
+ @Unstable
+ public YarnApplicationAttemptState getYarnApplicationAttemptState() {
+ return yarnApplicationAttemptState;
+ }
+
+ @Public
+ @Unstable
+ public void setYarnApplicationAttemptState(
+ YarnApplicationAttemptState yarnApplicationAttemptState) {
+ this.yarnApplicationAttemptState = yarnApplicationAttemptState;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/records/ApplicationAttemptStartData.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/records/ApplicationAttemptStartData.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/records/ApplicationAttemptStartData.java
new file mode 100644
index 0000000..5131b20
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/records/ApplicationAttemptStartData.java
@@ -0,0 +1,82 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.metrics.records;
+
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.util.Records;
+
+/**
+ * The class contains the fields that can be determined when
+ * <code>RMAppAttempt</code> starts, and that need to be stored persistently.
+ */
+@Public
+@Unstable
+public abstract class ApplicationAttemptStartData {
+
+ @Public
+ @Unstable
+ public static ApplicationAttemptStartData newInstance(
+ ApplicationAttemptId appAttemptId, String host, int rpcPort,
+ ContainerId masterContainerId) {
+ ApplicationAttemptStartData appAttemptSD =
+ Records.newRecord(ApplicationAttemptStartData.class);
+ appAttemptSD.setApplicationAttemptId(appAttemptId);
+ appAttemptSD.setHost(host);
+ appAttemptSD.setRPCPort(rpcPort);
+ appAttemptSD.setMasterContainerId(masterContainerId);
+ return appAttemptSD;
+ }
+
+ @Public
+ @Unstable
+ public abstract ApplicationAttemptId getApplicationAttemptId();
+
+ @Public
+ @Unstable
+ public abstract void setApplicationAttemptId(
+ ApplicationAttemptId applicationAttemptId);
+
+ @Public
+ @Unstable
+ public abstract String getHost();
+
+ @Public
+ @Unstable
+ public abstract void setHost(String host);
+
+ @Public
+ @Unstable
+ public abstract int getRPCPort();
+
+ @Public
+ @Unstable
+ public abstract void setRPCPort(int rpcPort);
+
+ @Public
+ @Unstable
+ public abstract ContainerId getMasterContainerId();
+
+ @Public
+ @Unstable
+ public abstract void setMasterContainerId(ContainerId masterContainerId);
+
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/records/ApplicationFinishData.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/records/ApplicationFinishData.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/records/ApplicationFinishData.java
new file mode 100644
index 0000000..106b344
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/records/ApplicationFinishData.java
@@ -0,0 +1,94 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.metrics.records;
+
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
+import org.apache.hadoop.yarn.api.records.YarnApplicationState;
+import org.apache.hadoop.yarn.util.Records;
+
+/**
+ * The class contains the fields that can be determined when <code>RMApp</code>
+ * finishes, and that need to be stored persistently.
+ */
+@Public
+@Unstable
+public abstract class ApplicationFinishData {
+
+ @Public
+ @Unstable
+ public static ApplicationFinishData newInstance(ApplicationId applicationId,
+ long finishTime, String diagnosticsInfo,
+ FinalApplicationStatus finalApplicationStatus,
+ YarnApplicationState yarnApplicationState) {
+ ApplicationFinishData appFD =
+ Records.newRecord(ApplicationFinishData.class);
+ appFD.setApplicationId(applicationId);
+ appFD.setFinishTime(finishTime);
+ appFD.setDiagnosticsInfo(diagnosticsInfo);
+ appFD.setFinalApplicationStatus(finalApplicationStatus);
+ appFD.setYarnApplicationState(yarnApplicationState);
+ return appFD;
+ }
+
+ @Public
+ @Unstable
+ public abstract ApplicationId getApplicationId();
+
+ @Public
+ @Unstable
+ public abstract void setApplicationId(ApplicationId applicationId);
+
+ @Public
+ @Unstable
+ public abstract long getFinishTime();
+
+ @Public
+ @Unstable
+ public abstract void setFinishTime(long finishTime);
+
+ @Public
+ @Unstable
+ public abstract String getDiagnosticsInfo();
+
+ @Public
+ @Unstable
+ public abstract void setDiagnosticsInfo(String diagnosticsInfo);
+
+ @Public
+ @Unstable
+ public abstract FinalApplicationStatus getFinalApplicationStatus();
+
+ @Public
+ @Unstable
+ public abstract void setFinalApplicationStatus(
+ FinalApplicationStatus finalApplicationStatus);
+
+ @Public
+ @Unstable
+ public abstract YarnApplicationState getYarnApplicationState();
+
+ @Public
+ @Unstable
+ public abstract void setYarnApplicationState(
+ YarnApplicationState yarnApplicationState);
+
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/records/ApplicationHistoryData.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/records/ApplicationHistoryData.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/records/ApplicationHistoryData.java
new file mode 100644
index 0000000..a7781b3
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/records/ApplicationHistoryData.java
@@ -0,0 +1,213 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.metrics.records;
+
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
+import org.apache.hadoop.yarn.api.records.YarnApplicationState;
+
+/**
+ * The class contains all the fields that are stored persistently for
+ * <code>RMApp</code>.
+ */
+@Public
+@Unstable
+public class ApplicationHistoryData {
+
+ private ApplicationId applicationId;
+
+ private String applicationName;
+
+ private String applicationType;
+
+ private String user;
+
+ private String queue;
+
+ private long submitTime;
+
+ private long startTime;
+
+ private long finishTime;
+
+ private String diagnosticsInfo;
+
+ private FinalApplicationStatus finalApplicationStatus;
+
+ private YarnApplicationState yarnApplicationState;
+
+ @Public
+ @Unstable
+ public static ApplicationHistoryData newInstance(ApplicationId applicationId,
+ String applicationName, String applicationType, String queue,
+ String user, long submitTime, long startTime, long finishTime,
+ String diagnosticsInfo, FinalApplicationStatus finalApplicationStatus,
+ YarnApplicationState yarnApplicationState) {
+ ApplicationHistoryData appHD = new ApplicationHistoryData();
+ appHD.setApplicationId(applicationId);
+ appHD.setApplicationName(applicationName);
+ appHD.setApplicationType(applicationType);
+ appHD.setQueue(queue);
+ appHD.setUser(user);
+ appHD.setSubmitTime(submitTime);
+ appHD.setStartTime(startTime);
+ appHD.setFinishTime(finishTime);
+ appHD.setDiagnosticsInfo(diagnosticsInfo);
+ appHD.setFinalApplicationStatus(finalApplicationStatus);
+ appHD.setYarnApplicationState(yarnApplicationState);
+ return appHD;
+ }
+
+ @Public
+ @Unstable
+ public ApplicationId getApplicationId() {
+ return applicationId;
+ }
+
+ @Public
+ @Unstable
+ public void setApplicationId(ApplicationId applicationId) {
+ this.applicationId = applicationId;
+ }
+
+ @Public
+ @Unstable
+ public String getApplicationName() {
+ return applicationName;
+ }
+
+ @Public
+ @Unstable
+ public void setApplicationName(String applicationName) {
+ this.applicationName = applicationName;
+ }
+
+ @Public
+ @Unstable
+ public String getApplicationType() {
+ return applicationType;
+ }
+
+ @Public
+ @Unstable
+ public void setApplicationType(String applicationType) {
+ this.applicationType = applicationType;
+ }
+
+ @Public
+ @Unstable
+ public String getUser() {
+ return user;
+ }
+
+ @Public
+ @Unstable
+ public void setUser(String user) {
+ this.user = user;
+ }
+
+ @Public
+ @Unstable
+ public String getQueue() {
+ return queue;
+ }
+
+ @Public
+ @Unstable
+ public void setQueue(String queue) {
+ this.queue = queue;
+ }
+
+ @Public
+ @Unstable
+ public long getSubmitTime() {
+ return submitTime;
+ }
+
+ @Public
+ @Unstable
+ public void setSubmitTime(long submitTime) {
+ this.submitTime = submitTime;
+ }
+
+ @Public
+ @Unstable
+ public long getStartTime() {
+ return startTime;
+ }
+
+ @Public
+ @Unstable
+ public void setStartTime(long startTime) {
+ this.startTime = startTime;
+ }
+
+ @Public
+ @Unstable
+ public long getFinishTime() {
+ return finishTime;
+ }
+
+ @Public
+ @Unstable
+ public void setFinishTime(long finishTime) {
+ this.finishTime = finishTime;
+ }
+
+ @Public
+ @Unstable
+ public String getDiagnosticsInfo() {
+ return diagnosticsInfo;
+ }
+
+ @Public
+ @Unstable
+ public void setDiagnosticsInfo(String diagnosticsInfo) {
+ this.diagnosticsInfo = diagnosticsInfo;
+ }
+
+ @Public
+ @Unstable
+ public FinalApplicationStatus getFinalApplicationStatus() {
+ return finalApplicationStatus;
+ }
+
+ @Public
+ @Unstable
+ public void setFinalApplicationStatus(
+ FinalApplicationStatus finalApplicationStatus) {
+ this.finalApplicationStatus = finalApplicationStatus;
+ }
+
+ @Public
+ @Unstable
+ public YarnApplicationState getYarnApplicationState() {
+ return this.yarnApplicationState;
+ }
+
+ @Public
+ @Unstable
+ public void
+ setYarnApplicationState(YarnApplicationState yarnApplicationState) {
+ this.yarnApplicationState = yarnApplicationState;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/records/ApplicationStartData.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/records/ApplicationStartData.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/records/ApplicationStartData.java
new file mode 100644
index 0000000..5d60545
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/records/ApplicationStartData.java
@@ -0,0 +1,106 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.metrics.records;
+
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.util.Records;
+
+/**
+ * The class contains the fields that can be determined when <code>RMApp</code>
+ * starts, and that need to be stored persistently.
+ */
+@Public
+@Unstable
+public abstract class ApplicationStartData {
+
+ @Public
+ @Unstable
+ public static ApplicationStartData newInstance(ApplicationId applicationId,
+ String applicationName, String applicationType, String queue,
+ String user, long submitTime, long startTime) {
+ ApplicationStartData appSD = Records.newRecord(ApplicationStartData.class);
+ appSD.setApplicationId(applicationId);
+ appSD.setApplicationName(applicationName);
+ appSD.setApplicationType(applicationType);
+ appSD.setQueue(queue);
+ appSD.setUser(user);
+ appSD.setSubmitTime(submitTime);
+ appSD.setStartTime(startTime);
+ return appSD;
+ }
+
+ @Public
+ @Unstable
+ public abstract ApplicationId getApplicationId();
+
+ @Public
+ @Unstable
+ public abstract void setApplicationId(ApplicationId applicationId);
+
+ @Public
+ @Unstable
+ public abstract String getApplicationName();
+
+ @Public
+ @Unstable
+ public abstract void setApplicationName(String applicationName);
+
+ @Public
+ @Unstable
+ public abstract String getApplicationType();
+
+ @Public
+ @Unstable
+ public abstract void setApplicationType(String applicationType);
+
+ @Public
+ @Unstable
+ public abstract String getUser();
+
+ @Public
+ @Unstable
+ public abstract void setUser(String user);
+
+ @Public
+ @Unstable
+ public abstract String getQueue();
+
+ @Public
+ @Unstable
+ public abstract void setQueue(String queue);
+
+ @Public
+ @Unstable
+ public abstract long getSubmitTime();
+
+ @Public
+ @Unstable
+ public abstract void setSubmitTime(long submitTime);
+
+ @Public
+ @Unstable
+ public abstract long getStartTime();
+
+ @Public
+ @Unstable
+ public abstract void setStartTime(long startTime);
+
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/records/ContainerFinishData.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/records/ContainerFinishData.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/records/ContainerFinishData.java
new file mode 100644
index 0000000..4ab4069
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/records/ContainerFinishData.java
@@ -0,0 +1,90 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.metrics.records;
+
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.ContainerState;
+import org.apache.hadoop.yarn.util.Records;
+
+/**
+ * The class contains the fields that can be determined when
+ * <code>RMContainer</code> finishes, and that need to be stored persistently.
+ */
+@Public
+@Unstable
+public abstract class ContainerFinishData {
+
+ @Public
+ @Unstable
+ public static ContainerFinishData newInstance(ContainerId containerId,
+ long finishTime, String diagnosticsInfo, int containerExitCode,
+ ContainerState containerState) {
+ ContainerFinishData containerFD =
+ Records.newRecord(ContainerFinishData.class);
+ containerFD.setContainerId(containerId);
+ containerFD.setFinishTime(finishTime);
+ containerFD.setDiagnosticsInfo(diagnosticsInfo);
+ containerFD.setContainerExitStatus(containerExitCode);
+ containerFD.setContainerState(containerState);
+ return containerFD;
+ }
+
+ @Public
+ @Unstable
+ public abstract ContainerId getContainerId();
+
+ @Public
+ @Unstable
+ public abstract void setContainerId(ContainerId containerId);
+
+ @Public
+ @Unstable
+ public abstract long getFinishTime();
+
+ @Public
+ @Unstable
+ public abstract void setFinishTime(long finishTime);
+
+ @Public
+ @Unstable
+ public abstract String getDiagnosticsInfo();
+
+ @Public
+ @Unstable
+ public abstract void setDiagnosticsInfo(String diagnosticsInfo);
+
+ @Public
+ @Unstable
+ public abstract int getContainerExitStatus();
+
+ @Public
+ @Unstable
+ public abstract void setContainerExitStatus(int containerExitStatus);
+
+ @Public
+ @Unstable
+ public abstract ContainerState getContainerState();
+
+ @Public
+ @Unstable
+ public abstract void setContainerState(ContainerState containerState);
+
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/records/ContainerHistoryData.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/records/ContainerHistoryData.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/records/ContainerHistoryData.java
new file mode 100644
index 0000000..13d792d
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/records/ContainerHistoryData.java
@@ -0,0 +1,182 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.metrics.records;
+
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.ContainerState;
+import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.api.records.Priority;
+import org.apache.hadoop.yarn.api.records.Resource;
+
+/**
+ * The class contains all the fields that are stored persistently for
+ * <code>RMContainer</code>.
+ */
+@Public
+@Unstable
+public class ContainerHistoryData {
+
+ private ContainerId containerId;
+
+ private Resource allocatedResource;
+
+ private NodeId assignedNode;
+
+ private Priority priority;
+
+ private long startTime;
+
+ private long finishTime;
+
+ private String diagnosticsInfo;
+
+ private int containerExitStatus;
+
+ private ContainerState containerState;
+
+ @Public
+ @Unstable
+ public static ContainerHistoryData newInstance(ContainerId containerId,
+ Resource allocatedResource, NodeId assignedNode, Priority priority,
+ long startTime, long finishTime, String diagnosticsInfo,
+ int containerExitCode, ContainerState containerState) {
+ ContainerHistoryData containerHD = new ContainerHistoryData();
+ containerHD.setContainerId(containerId);
+ containerHD.setAllocatedResource(allocatedResource);
+ containerHD.setAssignedNode(assignedNode);
+ containerHD.setPriority(priority);
+ containerHD.setStartTime(startTime);
+ containerHD.setFinishTime(finishTime);
+ containerHD.setDiagnosticsInfo(diagnosticsInfo);
+ containerHD.setContainerExitStatus(containerExitCode);
+ containerHD.setContainerState(containerState);
+ return containerHD;
+ }
+
+ @Public
+ @Unstable
+ public ContainerId getContainerId() {
+ return containerId;
+ }
+
+ @Public
+ @Unstable
+ public void setContainerId(ContainerId containerId) {
+ this.containerId = containerId;
+ }
+
+ @Public
+ @Unstable
+ public Resource getAllocatedResource() {
+ return allocatedResource;
+ }
+
+ @Public
+ @Unstable
+ public void setAllocatedResource(Resource resource) {
+ this.allocatedResource = resource;
+ }
+
+ @Public
+ @Unstable
+ public NodeId getAssignedNode() {
+ return assignedNode;
+ }
+
+ @Public
+ @Unstable
+ public void setAssignedNode(NodeId nodeId) {
+ this.assignedNode = nodeId;
+ }
+
+ @Public
+ @Unstable
+ public Priority getPriority() {
+ return priority;
+ }
+
+ @Public
+ @Unstable
+ public void setPriority(Priority priority) {
+ this.priority = priority;
+ }
+
+ @Public
+ @Unstable
+ public long getStartTime() {
+ return startTime;
+ }
+
+ @Public
+ @Unstable
+ public void setStartTime(long startTime) {
+ this.startTime = startTime;
+ }
+
+ @Public
+ @Unstable
+ public long getFinishTime() {
+ return finishTime;
+ }
+
+ @Public
+ @Unstable
+ public void setFinishTime(long finishTime) {
+ this.finishTime = finishTime;
+ }
+
+ @Public
+ @Unstable
+ public String getDiagnosticsInfo() {
+ return diagnosticsInfo;
+ }
+
+ @Public
+ @Unstable
+ public void setDiagnosticsInfo(String diagnosticsInfo) {
+ this.diagnosticsInfo = diagnosticsInfo;
+ }
+
+ @Public
+ @Unstable
+ public int getContainerExitStatus() {
+ return containerExitStatus;
+ }
+
+ @Public
+ @Unstable
+ public void setContainerExitStatus(int containerExitStatus) {
+ this.containerExitStatus = containerExitStatus;
+ }
+
+ @Public
+ @Unstable
+ public ContainerState getContainerState() {
+ return containerState;
+ }
+
+ @Public
+ @Unstable
+ public void setContainerState(ContainerState containerState) {
+ this.containerState = containerState;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/records/ContainerStartData.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/records/ContainerStartData.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/records/ContainerStartData.java
new file mode 100644
index 0000000..6adade1
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/records/ContainerStartData.java
@@ -0,0 +1,92 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.metrics.records;
+
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.api.records.Priority;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.util.Records;
+
+/**
+ * The class contains the fields that can be determined when
+ * <code>RMContainer</code> starts, and that need to be stored persistently.
+ */
+@Public
+@Unstable
+public abstract class ContainerStartData {
+
+ @Public
+ @Unstable
+ public static ContainerStartData newInstance(ContainerId containerId,
+ Resource allocatedResource, NodeId assignedNode, Priority priority,
+ long startTime) {
+ ContainerStartData containerSD =
+ Records.newRecord(ContainerStartData.class);
+ containerSD.setContainerId(containerId);
+ containerSD.setAllocatedResource(allocatedResource);
+ containerSD.setAssignedNode(assignedNode);
+ containerSD.setPriority(priority);
+ containerSD.setStartTime(startTime);
+ return containerSD;
+ }
+
+ @Public
+ @Unstable
+ public abstract ContainerId getContainerId();
+
+ @Public
+ @Unstable
+ public abstract void setContainerId(ContainerId containerId);
+
+ @Public
+ @Unstable
+ public abstract Resource getAllocatedResource();
+
+ @Public
+ @Unstable
+ public abstract void setAllocatedResource(Resource resource);
+
+ @Public
+ @Unstable
+ public abstract NodeId getAssignedNode();
+
+ @Public
+ @Unstable
+ public abstract void setAssignedNode(NodeId nodeId);
+
+ @Public
+ @Unstable
+ public abstract Priority getPriority();
+
+ @Public
+ @Unstable
+ public abstract void setPriority(Priority priority);
+
+ @Public
+ @Unstable
+ public abstract long getStartTime();
+
+ @Public
+ @Unstable
+ public abstract void setStartTime(long startTime);
+
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/records/impl/pb/ApplicationAttemptFinishDataPBImpl.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/records/impl/pb/ApplicationAttemptFinishDataPBImpl.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/records/impl/pb/ApplicationAttemptFinishDataPBImpl.java
new file mode 100644
index 0000000..cb0e545
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/records/impl/pb/ApplicationAttemptFinishDataPBImpl.java
@@ -0,0 +1,239 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.metrics.records.impl.pb;
+
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
+import org.apache.hadoop.yarn.api.records.YarnApplicationAttemptState;
+import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationAttemptIdPBImpl;
+import org.apache.hadoop.yarn.api.records.impl.pb.ProtoUtils;
+import org.apache.hadoop.yarn.proto.ApplicationHistoryServerProtos.ApplicationAttemptFinishDataProto;
+import org.apache.hadoop.yarn.proto.ApplicationHistoryServerProtos.ApplicationAttemptFinishDataProtoOrBuilder;
+import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationAttemptIdProto;
+import org.apache.hadoop.yarn.proto.YarnProtos.FinalApplicationStatusProto;
+import org.apache.hadoop.yarn.proto.YarnProtos.YarnApplicationAttemptStateProto;
+import org.apache.ambari.metrics.records.ApplicationAttemptFinishData;
+
+import com.google.protobuf.TextFormat;
+
+public class ApplicationAttemptFinishDataPBImpl extends
+ ApplicationAttemptFinishData {
+
+ ApplicationAttemptFinishDataProto proto = ApplicationAttemptFinishDataProto
+ .getDefaultInstance();
+ ApplicationAttemptFinishDataProto.Builder builder = null;
+ boolean viaProto = false;
+
+ public ApplicationAttemptFinishDataPBImpl() {
+ builder = ApplicationAttemptFinishDataProto.newBuilder();
+ }
+
+ public ApplicationAttemptFinishDataPBImpl(
+ ApplicationAttemptFinishDataProto proto) {
+ this.proto = proto;
+ viaProto = true;
+ }
+
+ private ApplicationAttemptId applicationAttemptId;
+
+ @Override
+ public ApplicationAttemptId getApplicationAttemptId() {
+ if (this.applicationAttemptId != null) {
+ return this.applicationAttemptId;
+ }
+ ApplicationAttemptFinishDataProtoOrBuilder p = viaProto ? proto : builder;
+ if (!p.hasApplicationAttemptId()) {
+ return null;
+ }
+ this.applicationAttemptId =
+ convertFromProtoFormat(p.getApplicationAttemptId());
+ return this.applicationAttemptId;
+ }
+
+ @Override
+ public void
+ setApplicationAttemptId(ApplicationAttemptId applicationAttemptId) {
+ maybeInitBuilder();
+ if (applicationAttemptId == null) {
+ builder.clearApplicationAttemptId();
+ }
+ this.applicationAttemptId = applicationAttemptId;
+ }
+
+ @Override
+ public String getTrackingURL() {
+ ApplicationAttemptFinishDataProtoOrBuilder p = viaProto ? proto : builder;
+ if (!p.hasTrackingUrl()) {
+ return null;
+ }
+ return p.getTrackingUrl();
+ }
+
+ @Override
+ public void setTrackingURL(String trackingURL) {
+ maybeInitBuilder();
+ if (trackingURL == null) {
+ builder.clearTrackingUrl();
+ return;
+ }
+ builder.setTrackingUrl(trackingURL);
+ }
+
+ @Override
+ public String getDiagnosticsInfo() {
+ ApplicationAttemptFinishDataProtoOrBuilder p = viaProto ? proto : builder;
+ if (!p.hasDiagnosticsInfo()) {
+ return null;
+ }
+ return p.getDiagnosticsInfo();
+ }
+
+ @Override
+ public void setDiagnosticsInfo(String diagnosticsInfo) {
+ maybeInitBuilder();
+ if (diagnosticsInfo == null) {
+ builder.clearDiagnosticsInfo();
+ return;
+ }
+ builder.setDiagnosticsInfo(diagnosticsInfo);
+ }
+
+ @Override
+ public FinalApplicationStatus getFinalApplicationStatus() {
+ ApplicationAttemptFinishDataProtoOrBuilder p = viaProto ? proto : builder;
+ if (!p.hasFinalApplicationStatus()) {
+ return null;
+ }
+ return convertFromProtoFormat(p.getFinalApplicationStatus());
+ }
+
+ @Override
+ public void setFinalApplicationStatus(
+ FinalApplicationStatus finalApplicationStatus) {
+ maybeInitBuilder();
+ if (finalApplicationStatus == null) {
+ builder.clearFinalApplicationStatus();
+ return;
+ }
+ builder
+ .setFinalApplicationStatus(convertToProtoFormat(finalApplicationStatus));
+ }
+
+ @Override
+ public YarnApplicationAttemptState getYarnApplicationAttemptState() {
+ ApplicationAttemptFinishDataProtoOrBuilder p = viaProto ? proto : builder;
+ if (!p.hasYarnApplicationAttemptState()) {
+ return null;
+ }
+ return convertFromProtoFormat(p.getYarnApplicationAttemptState());
+ }
+
+ @Override
+ public void setYarnApplicationAttemptState(YarnApplicationAttemptState state) {
+ maybeInitBuilder();
+ if (state == null) {
+ builder.clearYarnApplicationAttemptState();
+ return;
+ }
+ builder.setYarnApplicationAttemptState(convertToProtoFormat(state));
+ }
+
+ public ApplicationAttemptFinishDataProto getProto() {
+ mergeLocalToProto();
+ proto = viaProto ? proto : builder.build();
+ viaProto = true;
+ return proto;
+ }
+
+ @Override
+ public int hashCode() {
+ return getProto().hashCode();
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (other == null)
+ return false;
+ if (other.getClass().isAssignableFrom(this.getClass())) {
+ return this.getProto().equals(this.getClass().cast(other).getProto());
+ }
+ return false;
+ }
+
+ @Override
+ public String toString() {
+ return TextFormat.shortDebugString(getProto());
+ }
+
+ private void mergeLocalToBuilder() {
+ if (this.applicationAttemptId != null
+ && !((ApplicationAttemptIdPBImpl) this.applicationAttemptId).getProto()
+ .equals(builder.getApplicationAttemptId())) {
+ builder
+ .setApplicationAttemptId(convertToProtoFormat(this.applicationAttemptId));
+ }
+ }
+
+ private void mergeLocalToProto() {
+ if (viaProto) {
+ maybeInitBuilder();
+ }
+ mergeLocalToBuilder();
+ proto = builder.build();
+ viaProto = true;
+ }
+
+ private void maybeInitBuilder() {
+ if (viaProto || builder == null) {
+ builder = ApplicationAttemptFinishDataProto.newBuilder(proto);
+ }
+ viaProto = false;
+ }
+
+ private ApplicationAttemptIdPBImpl convertFromProtoFormat(
+ ApplicationAttemptIdProto applicationAttemptId) {
+ return new ApplicationAttemptIdPBImpl(applicationAttemptId);
+ }
+
+ private ApplicationAttemptIdProto convertToProtoFormat(
+ ApplicationAttemptId applicationAttemptId) {
+ return ((ApplicationAttemptIdPBImpl) applicationAttemptId).getProto();
+ }
+
+ private FinalApplicationStatus convertFromProtoFormat(
+ FinalApplicationStatusProto finalApplicationStatus) {
+ return ProtoUtils.convertFromProtoFormat(finalApplicationStatus);
+ }
+
+ private FinalApplicationStatusProto convertToProtoFormat(
+ FinalApplicationStatus finalApplicationStatus) {
+ return ProtoUtils.convertToProtoFormat(finalApplicationStatus);
+ }
+
+ private YarnApplicationAttemptStateProto convertToProtoFormat(
+ YarnApplicationAttemptState state) {
+ return ProtoUtils.convertToProtoFormat(state);
+ }
+
+ private YarnApplicationAttemptState convertFromProtoFormat(
+ YarnApplicationAttemptStateProto yarnApplicationAttemptState) {
+ return ProtoUtils.convertFromProtoFormat(yarnApplicationAttemptState);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/records/impl/pb/ApplicationAttemptStartDataPBImpl.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/records/impl/pb/ApplicationAttemptStartDataPBImpl.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/records/impl/pb/ApplicationAttemptStartDataPBImpl.java
new file mode 100644
index 0000000..6c76d3c
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/records/impl/pb/ApplicationAttemptStartDataPBImpl.java
@@ -0,0 +1,208 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.metrics.records.impl.pb;
+
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationAttemptIdPBImpl;
+import org.apache.hadoop.yarn.api.records.impl.pb.ContainerIdPBImpl;
+import org.apache.hadoop.yarn.proto.ApplicationHistoryServerProtos.ApplicationAttemptStartDataProto;
+import org.apache.hadoop.yarn.proto.ApplicationHistoryServerProtos.ApplicationAttemptStartDataProtoOrBuilder;
+import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationAttemptIdProto;
+import org.apache.hadoop.yarn.proto.YarnProtos.ContainerIdProto;
+import org.apache.ambari.metrics.records.ApplicationAttemptStartData;
+
+import com.google.protobuf.TextFormat;
+
+public class ApplicationAttemptStartDataPBImpl extends
+ ApplicationAttemptStartData {
+
+ ApplicationAttemptStartDataProto proto = ApplicationAttemptStartDataProto
+ .getDefaultInstance();
+ ApplicationAttemptStartDataProto.Builder builder = null;
+ boolean viaProto = false;
+
+ public ApplicationAttemptStartDataPBImpl() {
+ builder = ApplicationAttemptStartDataProto.newBuilder();
+ }
+
+ public ApplicationAttemptStartDataPBImpl(
+ ApplicationAttemptStartDataProto proto) {
+ this.proto = proto;
+ viaProto = true;
+ }
+
+ private ApplicationAttemptId applicationAttemptId;
+ private ContainerId masterContainerId;
+
+ @Override
+ public ApplicationAttemptId getApplicationAttemptId() {
+ if (this.applicationAttemptId != null) {
+ return this.applicationAttemptId;
+ }
+ ApplicationAttemptStartDataProtoOrBuilder p = viaProto ? proto : builder;
+ if (!p.hasApplicationAttemptId()) {
+ return null;
+ }
+ this.applicationAttemptId =
+ convertFromProtoFormat(p.getApplicationAttemptId());
+ return this.applicationAttemptId;
+ }
+
+ @Override
+ public void
+ setApplicationAttemptId(ApplicationAttemptId applicationAttemptId) {
+ maybeInitBuilder();
+ if (applicationAttemptId == null) {
+ builder.clearApplicationAttemptId();
+ }
+ this.applicationAttemptId = applicationAttemptId;
+ }
+
+ @Override
+ public String getHost() {
+ ApplicationAttemptStartDataProtoOrBuilder p = viaProto ? proto : builder;
+ if (!p.hasHost()) {
+ return null;
+ }
+ return p.getHost();
+ }
+
+ @Override
+ public void setHost(String host) {
+ maybeInitBuilder();
+ if (host == null) {
+ builder.clearHost();
+ return;
+ }
+ builder.setHost(host);
+ }
+
+ @Override
+ public int getRPCPort() {
+ ApplicationAttemptStartDataProtoOrBuilder p = viaProto ? proto : builder;
+ return p.getRpcPort();
+ }
+
+ @Override
+ public void setRPCPort(int rpcPort) {
+ maybeInitBuilder();
+ builder.setRpcPort(rpcPort);
+ }
+
+ @Override
+ public ContainerId getMasterContainerId() {
+ if (this.masterContainerId != null) {
+ return this.masterContainerId;
+ }
+ ApplicationAttemptStartDataProtoOrBuilder p = viaProto ? proto : builder;
+ if (!p.hasApplicationAttemptId()) {
+ return null;
+ }
+ this.masterContainerId = convertFromProtoFormat(p.getMasterContainerId());
+ return this.masterContainerId;
+ }
+
+ @Override
+ public void setMasterContainerId(ContainerId masterContainerId) {
+ maybeInitBuilder();
+ if (masterContainerId == null) {
+ builder.clearMasterContainerId();
+ }
+ this.masterContainerId = masterContainerId;
+ }
+
+ public ApplicationAttemptStartDataProto getProto() {
+ mergeLocalToProto();
+ proto = viaProto ? proto : builder.build();
+ viaProto = true;
+ return proto;
+ }
+
+ @Override
+ public int hashCode() {
+ return getProto().hashCode();
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (other == null)
+ return false;
+ if (other.getClass().isAssignableFrom(this.getClass())) {
+ return this.getProto().equals(this.getClass().cast(other).getProto());
+ }
+ return false;
+ }
+
+ @Override
+ public String toString() {
+ return TextFormat.shortDebugString(getProto());
+ }
+
+ private void mergeLocalToBuilder() {
+ if (this.applicationAttemptId != null
+ && !((ApplicationAttemptIdPBImpl) this.applicationAttemptId).getProto()
+ .equals(builder.getApplicationAttemptId())) {
+ builder
+ .setApplicationAttemptId(convertToProtoFormat(this.applicationAttemptId));
+ }
+ if (this.masterContainerId != null
+ && !((ContainerIdPBImpl) this.masterContainerId).getProto().equals(
+ builder.getMasterContainerId())) {
+ builder
+ .setMasterContainerId(convertToProtoFormat(this.masterContainerId));
+ }
+ }
+
+ private void mergeLocalToProto() {
+ if (viaProto) {
+ maybeInitBuilder();
+ }
+ mergeLocalToBuilder();
+ proto = builder.build();
+ viaProto = true;
+ }
+
+ private void maybeInitBuilder() {
+ if (viaProto || builder == null) {
+ builder = ApplicationAttemptStartDataProto.newBuilder(proto);
+ }
+ viaProto = false;
+ }
+
+ private ApplicationAttemptIdPBImpl convertFromProtoFormat(
+ ApplicationAttemptIdProto applicationAttemptId) {
+ return new ApplicationAttemptIdPBImpl(applicationAttemptId);
+ }
+
+ private ApplicationAttemptIdProto convertToProtoFormat(
+ ApplicationAttemptId applicationAttemptId) {
+ return ((ApplicationAttemptIdPBImpl) applicationAttemptId).getProto();
+ }
+
+ private ContainerIdPBImpl
+ convertFromProtoFormat(ContainerIdProto containerId) {
+ return new ContainerIdPBImpl(containerId);
+ }
+
+ private ContainerIdProto convertToProtoFormat(ContainerId masterContainerId) {
+ return ((ContainerIdPBImpl) masterContainerId).getProto();
+ }
+
+}
[08/28] ambari git commit: AMBARI-22740 : Fix integration test for
HBase in branch-3.0-ams due to UUID changes. (avijayan)
Posted by av...@apache.org.
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/timeline/HBaseTimelineMetricsServiceTest.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/timeline/HBaseTimelineMetricsServiceTest.java b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/timeline/HBaseTimelineMetricsServiceTest.java
new file mode 100644
index 0000000..514bf86
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/timeline/HBaseTimelineMetricsServiceTest.java
@@ -0,0 +1,136 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.metrics.core.timeline;
+
+import com.google.common.collect.Multimap;
+import junit.framework.Assert;
+import org.apache.ambari.metrics.core.timeline.aggregators.Function;
+import org.junit.Test;
+
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.TreeMap;
+
+import static org.apache.ambari.metrics.core.timeline.aggregators.Function.ReadFunction.AVG;
+import static org.apache.ambari.metrics.core.timeline.aggregators.Function.ReadFunction.SUM;
+import static org.apache.ambari.metrics.core.timeline.aggregators.Function.PostProcessingFunction.RATE;
+import static org.assertj.core.api.Assertions.assertThat;
+
+public class HBaseTimelineMetricsServiceTest {
+
+ public static final String MEM_METRIC = "mem";
+ public static final String BYTES_IN_METRIC = "bytes_in";
+ public static final String BYTES_NOT_AFUNCTION_METRIC = "bytes._not._afunction";
+
+ @Test
+ public void testParseMetricNamesToAggregationFunctions() throws Exception {
+ //giwen
+ List<String> metricNames = Arrays.asList(
+ MEM_METRIC + "._avg",
+ MEM_METRIC + "._sum",
+ MEM_METRIC + "._rate._avg",
+ BYTES_IN_METRIC,
+ BYTES_NOT_AFUNCTION_METRIC);
+
+ //when
+ Multimap<String, List<Function>> multimap =
+ HBaseTimelineMetricsService.parseMetricNamesToAggregationFunctions(metricNames);
+
+ //then
+ Assert.assertEquals(multimap.keySet().size(), 3);
+ Assert.assertTrue(multimap.containsKey(MEM_METRIC));
+ Assert.assertTrue(multimap.containsKey(BYTES_IN_METRIC));
+ Assert.assertTrue(multimap.containsKey(BYTES_NOT_AFUNCTION_METRIC));
+
+ List<List<Function>> metricEntry = (List<List<Function>>) multimap.get(MEM_METRIC);
+ HashMap<String, List<Function>> mfm = new HashMap<String, List<Function>>();
+ mfm.put(MEM_METRIC, metricEntry.get(0));
+
+ assertThat(mfm.get(MEM_METRIC)).containsOnly(
+ new Function(AVG, null));
+
+ mfm = new HashMap<String, List<Function>>();
+ mfm.put(MEM_METRIC, metricEntry.get(1));
+ assertThat(mfm.get(MEM_METRIC)).containsOnly(
+ new Function(SUM, null));
+
+ mfm = new HashMap<String, List<Function>>();
+ mfm.put(MEM_METRIC, metricEntry.get(2));
+ assertThat(mfm.get(MEM_METRIC)).containsOnly(
+ new Function(AVG, RATE));
+
+ metricEntry = (List<List<Function>>) multimap.get(BYTES_IN_METRIC);
+ mfm = new HashMap<String, List<Function>>();
+ mfm.put(BYTES_IN_METRIC, metricEntry.get(0));
+
+ assertThat(mfm.get(BYTES_IN_METRIC))
+ .contains(Function.DEFAULT_VALUE_FUNCTION);
+
+ metricEntry = (List<List<Function>>) multimap.get(BYTES_NOT_AFUNCTION_METRIC);
+ mfm = new HashMap<String, List<Function>>();
+ mfm.put(BYTES_NOT_AFUNCTION_METRIC, metricEntry.get(0));
+
+ assertThat(mfm.get(BYTES_NOT_AFUNCTION_METRIC))
+ .contains(Function.DEFAULT_VALUE_FUNCTION);
+
+ }
+
+ @Test
+ public void testRateCalculationOnMetricsWithEqualValues() throws Exception {
+ Map<Long, Double> metricValues = new TreeMap<>();
+ metricValues.put(1454000000000L, 1.0);
+ metricValues.put(1454000001000L, 6.0);
+ metricValues.put(1454000002000L, 0.0);
+ metricValues.put(1454000003000L, 3.0);
+ metricValues.put(1454000004000L, 4.0);
+ metricValues.put(1454000005000L, 7.0);
+
+ // Calculate rate
+ Map<Long, Double> rates = HBaseTimelineMetricsService.updateValuesAsRate(new TreeMap<>(metricValues), false);
+
+ // Make sure rate is zero
+ Assert.assertTrue(rates.size() == 4);
+
+ Assert.assertFalse(rates.containsKey(1454000000000L));
+ Assert.assertFalse(rates.containsKey(1454000002000L));
+
+ Assert.assertEquals(rates.get(1454000001000L), 5.0);
+ Assert.assertEquals(rates.get(1454000003000L), 3.0);
+ Assert.assertEquals(rates.get(1454000004000L), 1.0);
+ Assert.assertEquals(rates.get(1454000005000L), 3.0);
+ }
+
+ @Test
+ public void testDiffCalculation() throws Exception {
+ Map<Long, Double> metricValues = new TreeMap<>();
+ metricValues.put(1454016368371L, 1011.25);
+ metricValues.put(1454016428371L, 1010.25);
+ metricValues.put(1454016488371L, 1012.25);
+ metricValues.put(1454016548371L, 1015.25);
+ metricValues.put(1454016608371L, 1020.25);
+
+ Map<Long, Double> rates = HBaseTimelineMetricsService.updateValuesAsRate(new TreeMap<>(metricValues), true);
+
+ Assert.assertTrue(rates.size() == 3);
+ Assert.assertTrue(rates.containsValue(2.0));
+ Assert.assertTrue(rates.containsValue(3.0));
+ Assert.assertTrue(rates.containsValue(5.0));
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/timeline/ITPhoenixHBaseAccessor.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/timeline/ITPhoenixHBaseAccessor.java b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/timeline/ITPhoenixHBaseAccessor.java
new file mode 100644
index 0000000..d4171bc
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/timeline/ITPhoenixHBaseAccessor.java
@@ -0,0 +1,472 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.metrics.core.timeline;
+
+import static junit.framework.Assert.assertEquals;
+import static junit.framework.Assert.assertTrue;
+import static org.apache.ambari.metrics.core.timeline.MetricTestHelper.createEmptyTimelineClusterMetric;
+import static org.apache.ambari.metrics.core.timeline.MetricTestHelper.prepareSingleTimelineMetric;
+import static org.apache.ambari.metrics.core.timeline.PhoenixHBaseAccessor.DATE_TIERED_COMPACTION_POLICY;
+import static org.apache.ambari.metrics.core.timeline.PhoenixHBaseAccessor.FIFO_COMPACTION_POLICY_CLASS;
+import static org.apache.ambari.metrics.core.timeline.PhoenixHBaseAccessor.HSTORE_COMPACTION_CLASS_KEY;
+import static org.apache.ambari.metrics.core.timeline.PhoenixHBaseAccessor.HSTORE_ENGINE_CLASS;
+import static org.apache.ambari.metrics.core.timeline.query.PhoenixTransactSQL.METRICS_AGGREGATE_MINUTE_TABLE_NAME;
+import static org.apache.ambari.metrics.core.timeline.query.PhoenixTransactSQL.METRICS_RECORD_TABLE_NAME;
+import static org.apache.ambari.metrics.core.timeline.query.PhoenixTransactSQL.PHOENIX_TABLES;
+import static org.apache.ambari.metrics.core.timeline.query.PhoenixTransactSQL.PHOENIX_TABLES_REGEX_PATTERN_STRING;
+
+import java.io.IOException;
+import java.lang.reflect.Field;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
+import org.apache.hadoop.hbase.client.Durability;
+import org.apache.hadoop.hbase.client.TableDescriptor;
+import org.apache.hadoop.metrics2.sink.timeline.ContainerMetric;
+import org.apache.hadoop.metrics2.sink.timeline.MetricClusterAggregate;
+import org.apache.hadoop.metrics2.sink.timeline.MetricHostAggregate;
+import org.apache.hadoop.metrics2.sink.timeline.Precision;
+import org.apache.hadoop.metrics2.sink.timeline.TimelineMetric;
+import org.apache.hadoop.metrics2.sink.timeline.TimelineMetrics;
+import org.apache.ambari.metrics.core.timeline.aggregators.Function;
+import org.apache.ambari.metrics.core.timeline.aggregators.TimelineClusterMetric;
+import org.apache.ambari.metrics.core.timeline.aggregators.TimelineMetricAggregator;
+import org.apache.ambari.metrics.core.timeline.aggregators.TimelineMetricAggregatorFactory;
+import org.apache.ambari.metrics.core.timeline.query.Condition;
+import org.apache.ambari.metrics.core.timeline.query.DefaultCondition;
+import org.junit.Test;
+
+import com.google.common.collect.ArrayListMultimap;
+import com.google.common.collect.Multimap;
+
+import junit.framework.Assert;
+
+
+
+public class ITPhoenixHBaseAccessor extends AbstractMiniHBaseClusterTest {
+
+ @Test
+ public void testGetMetricRecordsSeconds() throws IOException, SQLException {
+ // GIVEN
+ long startTime = System.currentTimeMillis();
+ long ctime = startTime;
+ long minute = 60 * 1000;
+ hdb.insertMetricRecords(MetricTestHelper.prepareSingleTimelineMetric(ctime, "local1",
+ "disk_free", 1), true);
+ hdb.insertMetricRecords(MetricTestHelper.prepareSingleTimelineMetric(ctime, "local2",
+ "disk_free", 2), true);
+ ctime += minute;
+ hdb.insertMetricRecords(MetricTestHelper.prepareSingleTimelineMetric(ctime, "local1",
+ "disk_free", 2), true);
+ hdb.insertMetricRecords(MetricTestHelper.prepareSingleTimelineMetric(ctime, "local2",
+ "disk_free", 1), true);
+
+ // WHEN
+ long endTime = ctime + minute;
+ List<byte[]> uuids = metadataManager.getUuids(new ArrayList<String>() {{ add("disk_free"); }},
+ Collections.singletonList("local1"),
+ "host", null);
+
+ Condition condition = new DefaultCondition(uuids,
+ new ArrayList<String>() {{ add("disk_free"); }},
+ Collections.singletonList("local1"),
+ "host", null, startTime, endTime, Precision.SECONDS, null, true);
+ TimelineMetrics timelineMetrics = hdb.getMetricRecords(condition,
+ singletonValueFunctionMap("disk_free"));
+
+ //THEN
+ assertEquals(1, timelineMetrics.getMetrics().size());
+ TimelineMetric metric = timelineMetrics.getMetrics().get(0);
+
+ assertEquals("disk_free", metric.getMetricName());
+ assertEquals("local1", metric.getHostName());
+ assertEquals(8, metric.getMetricValues().size());
+ }
+
+ @Test
+ public void testGetMetricRecordsMinutes() throws IOException, SQLException {
+ // GIVEN
+ TimelineMetricAggregator aggregatorMinute =
+ TimelineMetricAggregatorFactory.createTimelineMetricAggregatorMinute(hdb, new Configuration(), metadataManager, null);
+
+ long startTime = System.currentTimeMillis();
+ long ctime = startTime;
+ long minute = 60 * 1000;
+
+ TimelineMetrics metrics1 = MetricTestHelper.prepareSingleTimelineMetric(ctime, "local1",
+ "disk_free", 1);
+ hdb.insertMetricRecords(metrics1, true);
+
+ TimelineMetrics metrics2 = MetricTestHelper.prepareSingleTimelineMetric(ctime + minute, "local1",
+ "disk_free", 2);
+ hdb.insertMetricRecords(metrics2, true);
+
+ hdb.insertMetricRecords(MetricTestHelper.prepareSingleTimelineMetric(ctime, "local2",
+ "disk_free", 2), true);
+ long endTime = ctime + minute;
+ boolean success = aggregatorMinute.doWork(startTime - 1000, endTime);
+ assertTrue(success);
+
+ // WHEN
+ List<byte[]> uuids = metadataManager.getUuids(new ArrayList<String>() {{ add("disk_%"); }},
+ Collections.singletonList("local1"),
+ "host", null);
+ Condition condition = new DefaultCondition(uuids,
+ new ArrayList<String>() {{ add("disk_free"); }},
+ Collections.singletonList("local1"),
+ "host", null, startTime, endTime + 1000, Precision.MINUTES, null, false);
+ TimelineMetrics timelineMetrics = hdb.getMetricRecords(condition,
+ singletonValueFunctionMap("disk_free"));
+
+ //THEN
+ assertEquals(1, timelineMetrics.getMetrics().size());
+ TimelineMetric metric = timelineMetrics.getMetrics().get(0);
+
+ assertEquals("disk_free", metric.getMetricName());
+ assertEquals("local1", metric.getHostName());
+ assertEquals(1, metric.getMetricValues().size());
+ Iterator<Map.Entry<Long, Double>> iterator = metric.getMetricValues().entrySet().iterator();
+ assertEquals(1.5, iterator.next().getValue(), 0.00001);
+ }
+
+ @Test
+ public void testGetMetricRecordsHours() throws IOException, SQLException {
+ // GIVEN
+ TimelineMetricAggregator aggregator =
+ TimelineMetricAggregatorFactory.createTimelineMetricAggregatorHourly(hdb, new Configuration(), metadataManager, null);
+
+ MetricHostAggregate expectedAggregate =
+ MetricTestHelper.createMetricHostAggregate(2.0, 0.0, 20, 15.0);
+ Map<TimelineMetric, MetricHostAggregate>
+ aggMap = new HashMap<TimelineMetric,
+ MetricHostAggregate>();
+
+ long startTime = System.currentTimeMillis();
+ int min_5 = 5 * 60 * 1000;
+ long ctime = startTime - min_5;
+ aggMap.put(MetricTestHelper.createEmptyTimelineMetric(ctime += min_5), expectedAggregate);
+ aggMap.put(MetricTestHelper.createEmptyTimelineMetric(ctime += min_5), expectedAggregate);
+ aggMap.put(MetricTestHelper.createEmptyTimelineMetric(ctime += min_5), expectedAggregate);
+ aggMap.put(MetricTestHelper.createEmptyTimelineMetric(ctime += min_5), expectedAggregate);
+ aggMap.put(MetricTestHelper.createEmptyTimelineMetric(ctime += min_5), expectedAggregate);
+ aggMap.put(MetricTestHelper.createEmptyTimelineMetric(ctime += min_5), expectedAggregate);
+ aggMap.put(MetricTestHelper.createEmptyTimelineMetric(ctime += min_5), expectedAggregate);
+ aggMap.put(MetricTestHelper.createEmptyTimelineMetric(ctime += min_5), expectedAggregate);
+ aggMap.put(MetricTestHelper.createEmptyTimelineMetric(ctime += min_5), expectedAggregate);
+ aggMap.put(MetricTestHelper.createEmptyTimelineMetric(ctime += min_5), expectedAggregate);
+ aggMap.put(MetricTestHelper.createEmptyTimelineMetric(ctime += min_5), expectedAggregate);
+ aggMap.put(MetricTestHelper.createEmptyTimelineMetric(ctime += min_5), expectedAggregate);
+
+ hdb.saveHostAggregateRecords(aggMap, METRICS_AGGREGATE_MINUTE_TABLE_NAME);
+ long endTime = ctime + min_5;
+ boolean success = aggregator.doWork(startTime - 1000, endTime);
+ assertTrue(success);
+
+ // WHEN
+ List<byte[]> uuids = metadataManager.getUuids(new ArrayList<String>() {{ add("disk_used"); }},
+ Collections.singletonList("test_host"),
+ "test_app", "test_instance");
+
+ Condition condition = new DefaultCondition(uuids,
+ new ArrayList<String>() {{ add("disk_used"); }},
+ Collections.singletonList("test_host"),
+ "test_app", "test_instance", startTime, endTime + 1000, Precision.HOURS, null, true);
+ TimelineMetrics timelineMetrics = hdb.getMetricRecords(condition,
+ singletonValueFunctionMap("disk_used"));
+
+ //THEN
+ assertEquals(1, timelineMetrics.getMetrics().size());
+ TimelineMetric metric = timelineMetrics.getMetrics().get(0);
+
+ assertEquals("disk_used", metric.getMetricName());
+ assertEquals("test_host", metric.getHostName());
+ assertEquals(1, metric.getMetricValues().size());
+ Iterator<Map.Entry<Long, Double>> iterator = metric.getMetricValues().entrySet().iterator();
+ assertEquals(0.75, iterator.next().getValue(), 0.00001);
+ }
+
+ @Test
+ public void testGetClusterMetricRecordsSeconds() throws Exception {
+ // GIVEN
+ TimelineMetricAggregator agg =
+ TimelineMetricAggregatorFactory.createTimelineClusterAggregatorSecond(
+ hdb, new Configuration(), metadataManager, null, null);
+
+ long startTime = System.currentTimeMillis();
+ long ctime = startTime + 1;
+ long minute = 60 * 1000;
+ hdb.insertMetricRecords(MetricTestHelper.prepareSingleTimelineMetric(ctime, "local1",
+ "disk_free", 1), true);
+ hdb.insertMetricRecords(MetricTestHelper.prepareSingleTimelineMetric(ctime, "local2",
+ "disk_free", 2), true);
+ ctime += minute;
+ hdb.insertMetricRecords(MetricTestHelper.prepareSingleTimelineMetric(ctime, "local1",
+ "disk_free", 2), true);
+ hdb.insertMetricRecords(MetricTestHelper.prepareSingleTimelineMetric(ctime, "local2",
+ "disk_free", 1), true);
+
+ long endTime = ctime + minute + 1;
+ boolean success = agg.doWork(startTime, endTime);
+ assertTrue(success);
+
+ // WHEN
+ List<byte[]> uuids = metadataManager.getUuids(new ArrayList<String>() {{ add("disk_free"); }},
+ null,
+ "host", null);
+
+ Condition condition = new DefaultCondition(uuids,
+ new ArrayList<String>() {{ add("disk_free"); }},
+ null, "host", null, startTime - 90000, endTime, Precision.SECONDS, null, true);
+ TimelineMetrics timelineMetrics = hdb.getAggregateMetricRecords(condition,
+ singletonValueFunctionMap("disk_free"));
+
+ //THEN
+ assertEquals(1, timelineMetrics.getMetrics().size());
+ TimelineMetric metric = timelineMetrics.getMetrics().get(0);
+
+ assertEquals("disk_free", metric.getMetricName());
+ assertEquals(5, metric.getMetricValues().size());
+ assertEquals(1.5, metric.getMetricValues().values().iterator().next(), 0.00001);
+ }
+
+ @Test
+ public void testGetClusterMetricRecordLatestWithFunction() throws Exception {
+ // GIVEN
+ TimelineMetricAggregator agg =
+ TimelineMetricAggregatorFactory.createTimelineClusterAggregatorSecond(hdb,
+ new Configuration(), metadataManager, null, null);
+
+ long startTime = System.currentTimeMillis();
+ long ctime = startTime + 1;
+ long minute = 60 * 1000;
+ hdb.insertMetricRecords(MetricTestHelper.prepareSingleTimelineMetric(ctime, "local1",
+ "disk_free", 1), true);
+ hdb.insertMetricRecords(MetricTestHelper.prepareSingleTimelineMetric(ctime, "local2",
+ "disk_free", 2), true);
+ ctime += minute;
+ hdb.insertMetricRecords(MetricTestHelper.prepareSingleTimelineMetric(ctime, "local1",
+ "disk_free", 2), true);
+ hdb.insertMetricRecords(MetricTestHelper.prepareSingleTimelineMetric(ctime, "local2",
+ "disk_free", 1), true);
+
+ long endTime = ctime + minute + 1;
+ boolean success = agg.doWork(startTime, endTime);
+ assertTrue(success);
+
+ // WHEN
+ List<byte[]> uuids = metadataManager.getUuids(new ArrayList<String>() {{ add("disk_free"); }},
+ null,
+ "host", null);
+
+ Condition condition = new DefaultCondition(uuids,
+ new ArrayList<String>() {{ add("disk_free"); }},
+ null, "host", null, null, null, Precision.SECONDS, null, true);
+
+ Multimap<String, List<Function>> mmap = ArrayListMultimap.create();
+ mmap.put("disk_free", Collections.singletonList(new Function(Function.ReadFunction.SUM, null)));
+ TimelineMetrics timelineMetrics = hdb.getAggregateMetricRecords(condition, mmap);
+
+ //THEN
+ assertEquals(1, timelineMetrics.getMetrics().size());
+ TimelineMetric metric = timelineMetrics.getMetrics().get(0);
+
+ assertEquals("disk_free._sum", metric.getMetricName());
+ assertEquals(1, metric.getMetricValues().size());
+ assertEquals(3, metric.getMetricValues().values().iterator().next().intValue());
+ }
+
+ @Test
+ public void testGetClusterMetricRecordsHours() throws Exception {
+ // GIVEN
+ TimelineMetricAggregator agg =
+ TimelineMetricAggregatorFactory.createTimelineClusterAggregatorHourly(hdb, new Configuration(), metadataManager, null);
+
+ long startTime = System.currentTimeMillis();
+ long ctime = startTime;
+ long minute = 60 * 1000;
+
+ Map<TimelineClusterMetric, MetricClusterAggregate> records =
+ new HashMap<TimelineClusterMetric, MetricClusterAggregate>();
+
+ records.put(MetricTestHelper.createEmptyTimelineClusterMetric(ctime),
+ new MetricClusterAggregate(4.0, 2, 0.0, 4.0, 0.0));
+ records.put(MetricTestHelper.createEmptyTimelineClusterMetric(ctime += minute),
+ new MetricClusterAggregate(4.0, 2, 0.0, 4.0, 0.0));
+ records.put(MetricTestHelper.createEmptyTimelineClusterMetric(ctime += minute),
+ new MetricClusterAggregate(4.0, 2, 0.0, 4.0, 0.0));
+ records.put(MetricTestHelper.createEmptyTimelineClusterMetric(ctime += minute),
+ new MetricClusterAggregate(4.0, 2, 0.0, 4.0, 0.0));
+
+ hdb.saveClusterAggregateRecords(records);
+ boolean success = agg.doWork(startTime, ctime + minute);
+ assertTrue(success);
+
+ // WHEN
+ List<byte[]> uuids = metadataManager.getUuids(new ArrayList<String>() {{ add("disk_used"); }},
+ null,
+ "test_app",
+ "instance_id");
+
+ Condition condition = new DefaultCondition( uuids,
+ new ArrayList<String>() {{ add("disk_used"); }},
+ null,
+ "test_app",
+ "instance_id",
+ startTime,
+ ctime + minute + 1000,
+ Precision.HOURS,
+ null,
+ true);
+ TimelineMetrics timelineMetrics = hdb.getAggregateMetricRecords(condition,
+ singletonValueFunctionMap("disk_used"));
+
+ // THEN
+ assertEquals(1, timelineMetrics.getMetrics().size());
+ TimelineMetric metric = timelineMetrics.getMetrics().get(0);
+
+ assertEquals("disk_used", metric.getMetricName());
+ assertEquals("test_app", metric.getAppId());
+ assertEquals(1, metric.getMetricValues().size());
+ assertEquals(2.0, metric.getMetricValues().values().iterator().next(), 0.00001);
+ }
+
+ @Test
+ public void testInitPoliciesAndTTL() throws Exception {
+ Admin hBaseAdmin = hdb.getHBaseAdmin();
+ int precisionTtl = 2 * 86400;
+
+ Field f = PhoenixHBaseAccessor.class.getDeclaredField("tableTTL");
+ f.setAccessible(true);
+ Map<String, Integer> precisionValues = (Map<String, Integer>) f.get(hdb);
+ precisionValues.put(METRICS_RECORD_TABLE_NAME, precisionTtl);
+ f.set(hdb, precisionValues);
+
+ Field f2 = PhoenixHBaseAccessor.class.getDeclaredField("timelineMetricsTablesDurability");
+ f2.setAccessible(true);
+ f2.set(hdb, "ASYNC_WAL");
+
+ hdb.initPoliciesAndTTL();
+
+ // Verify expected policies are set
+ boolean normalizerEnabled = false;
+ String precisionTableCompactionPolicy = null;
+ String aggregateTableCompactionPolicy = null;
+ boolean tableDurabilitySet = false;
+ for (int i = 0; i < 10; i++) {
+ LOG.warn("Policy check retry : " + i);
+ for (String tableName : PHOENIX_TABLES) {
+ TableName[] tableNames = hBaseAdmin.listTableNames(PHOENIX_TABLES_REGEX_PATTERN_STRING, false);
+ Optional<TableName> tableNameOptional = Arrays.stream(tableNames)
+ .filter(t -> tableName.equals(t.getNameAsString())).findFirst();
+
+ TableDescriptor tableDescriptor = hBaseAdmin.getTableDescriptor(tableNameOptional.get());
+
+ normalizerEnabled = tableDescriptor.isNormalizationEnabled();
+ tableDurabilitySet = (Durability.ASYNC_WAL.equals(tableDescriptor.getDurability()));
+ if (tableName.equals(METRICS_RECORD_TABLE_NAME)) {
+ precisionTableCompactionPolicy = tableDescriptor.getValue(HSTORE_COMPACTION_CLASS_KEY);
+ } else {
+ aggregateTableCompactionPolicy = tableDescriptor.getValue(HSTORE_ENGINE_CLASS);
+ }
+ LOG.debug("Table: " + tableName + ", normalizerEnabled = " + normalizerEnabled);
+ // Best effort for 20 seconds
+ if (normalizerEnabled || (precisionTableCompactionPolicy == null && aggregateTableCompactionPolicy == null)) {
+ Thread.sleep(2000l);
+ }
+ if (tableName.equals(METRICS_RECORD_TABLE_NAME)) {
+ for (ColumnFamilyDescriptor family : tableDescriptor.getColumnFamilies()) {
+ precisionTtl = family.getTimeToLive();
+ }
+ }
+ }
+ }
+
+ Assert.assertFalse("Normalizer disabled.", normalizerEnabled);
+ Assert.assertTrue("Durability Set.", tableDurabilitySet);
+ Assert.assertEquals("FIFO compaction policy is set for METRIC_RECORD.", FIFO_COMPACTION_POLICY_CLASS, precisionTableCompactionPolicy);
+ Assert.assertEquals("FIFO compaction policy is set for aggregate tables", DATE_TIERED_COMPACTION_POLICY, aggregateTableCompactionPolicy);
+ Assert.assertEquals("Precision TTL value as expected.", 86400, precisionTtl);
+
+ hBaseAdmin.close();
+ }
+
+ private Multimap<String, List<Function>> singletonValueFunctionMap(String metricName) {
+ Multimap<String, List<Function>> mmap = ArrayListMultimap.create();
+ mmap.put(metricName, Collections.singletonList(new Function()));
+ return mmap;
+ }
+
+ @Test
+ public void testInsertContainerMetrics() throws Exception {
+ ContainerMetric metric = new ContainerMetric();
+ metric.setContainerId("container_1450744875949_0001_01_000001");
+ metric.setHostName("host1");
+ metric.setPmemLimit(2048);
+ metric.setVmemLimit(2048);
+ metric.setPmemUsedAvg(1024);
+ metric.setPmemUsedMin(1024);
+ metric.setPmemUsedMax(1024);
+ metric.setLaunchDuration(2000);
+ metric.setLocalizationDuration(3000);
+ long startTime = System.currentTimeMillis();
+ long finishTime = startTime + 5000;
+ metric.setStartTime(startTime);
+ metric.setFinishTime(finishTime);
+ metric.setExitCode(0);
+ List<ContainerMetric> list = Arrays.asList(metric);
+ hdb.insertContainerMetrics(list);
+ PreparedStatement stmt = conn.prepareStatement("SELECT * FROM CONTAINER_METRICS");
+ ResultSet set = stmt.executeQuery();
+ // check each filed is set properly when read back.
+ boolean foundRecord = false;
+ while (set.next()) {
+ assertEquals("application_1450744875949_0001", set.getString("APP_ID"));
+ assertEquals("container_1450744875949_0001_01_000001", set.getString("CONTAINER_ID"));
+ assertEquals(new java.sql.Timestamp(startTime), set.getTimestamp("START_TIME"));
+ assertEquals(new java.sql.Timestamp(finishTime), set.getTimestamp("FINISH_TIME"));
+ assertEquals(5000, set.getLong("DURATION"));
+ assertEquals("host1", set.getString("HOSTNAME"));
+ assertEquals(0, set.getInt("EXIT_CODE"));
+ assertEquals(3000, set.getLong("LOCALIZATION_DURATION"));
+ assertEquals(2000, set.getLong("LAUNCH_DURATION"));
+ assertEquals((double)2, set.getDouble("MEM_REQUESTED_GB"));
+ assertEquals((double)2 * 5000, set.getDouble("MEM_REQUESTED_GB_MILLIS"));
+ assertEquals((double)2, set.getDouble("MEM_VIRTUAL_GB"));
+ assertEquals((double)1, set.getDouble("MEM_USED_GB_MIN"));
+ assertEquals((double)1, set.getDouble("MEM_USED_GB_MAX"));
+ assertEquals((double)1, set.getDouble("MEM_USED_GB_AVG"));
+ assertEquals((double)(2 - 1), set.getDouble("MEM_UNUSED_GB"));
+ assertEquals((double)(2-1) * 5000, set.getDouble("MEM_UNUSED_GB_MILLIS"));
+ foundRecord = true;
+ }
+ assertTrue(foundRecord);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/timeline/MetricTestHelper.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/timeline/MetricTestHelper.java b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/timeline/MetricTestHelper.java
new file mode 100644
index 0000000..a99d488
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/timeline/MetricTestHelper.java
@@ -0,0 +1,119 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.metrics.core.timeline;
+
+import org.apache.hadoop.metrics2.sink.timeline.MetricHostAggregate;
+import org.apache.hadoop.metrics2.sink.timeline.TimelineMetric;
+import org.apache.hadoop.metrics2.sink.timeline.TimelineMetrics;
+import org.apache.ambari.metrics.core.timeline.aggregators.TimelineClusterMetric;
+
+import java.util.Arrays;
+import java.util.TreeMap;
+
+public class MetricTestHelper {
+
+ public static MetricHostAggregate createMetricHostAggregate(double max, double min, int numberOfSamples, double sum) {
+ MetricHostAggregate expectedAggregate = new MetricHostAggregate();
+ expectedAggregate.setMax(max);
+ expectedAggregate.setMin(min);
+ expectedAggregate.setNumberOfSamples(numberOfSamples);
+ expectedAggregate.setSum(sum);
+
+ return expectedAggregate;
+ }
+
+ public static TimelineMetrics prepareSingleTimelineMetric(long startTime,
+ String host,
+ String metricName,
+ double val) {
+ return prepareSingleTimelineMetric(startTime, host, null, metricName, val);
+ }
+
+ public static TimelineMetrics prepareSingleTimelineMetric(long startTime,
+ String host,
+ String instanceId,
+ String metricName,
+ double val) {
+ TimelineMetrics m = new TimelineMetrics();
+ m.setMetrics(Arrays.asList(
+ createTimelineMetric(startTime, metricName, host, null, instanceId, val)));
+
+ return m;
+ }
+
+ public static TimelineMetrics prepareSingleTimelineMetric(long startTime,
+ String host,
+ String appId,
+ String instanceId,
+ String metricName,
+ double val) {
+ TimelineMetrics m = new TimelineMetrics();
+ m.setMetrics(Arrays.asList(
+ createTimelineMetric(startTime, metricName, host, appId, instanceId, val)));
+
+ return m;
+ }
+
+
+ public static TimelineMetric createTimelineMetric(long startTime,
+ String metricName,
+ String host,
+ String appId,
+ String instanceId,
+ double val) {
+ TimelineMetric m = new TimelineMetric();
+ m.setHostName(host);
+ m.setAppId(appId != null ? appId : "host");
+ m.setInstanceId(instanceId);
+ m.setMetricName(metricName);
+ m.setStartTime(startTime);
+ TreeMap<Long, Double> vals = new TreeMap<Long, Double>();
+ vals.put(startTime + 15000l, val);
+ vals.put(startTime + 30000l, val);
+ vals.put(startTime + 45000l, val);
+ vals.put(startTime + 60000l, val);
+
+ m.setMetricValues(vals);
+
+ return m;
+ }
+
+ public static TimelineMetric createEmptyTimelineMetric(long startTime) {
+ TimelineMetric metric = new TimelineMetric();
+ metric.setMetricName("disk_used");
+ metric.setAppId("test_app");
+ metric.setInstanceId("test_instance");
+ metric.setHostName("test_host");
+ metric.setStartTime(startTime);
+
+ return metric;
+ }
+
+ public static TimelineClusterMetric createEmptyTimelineClusterMetric(
+ String name, long startTime) {
+ TimelineClusterMetric metric = new TimelineClusterMetric(name,
+ "test_app", "instance_id", startTime);
+
+ return metric;
+ }
+
+ public static TimelineClusterMetric createEmptyTimelineClusterMetric(
+ long startTime) {
+ return createEmptyTimelineClusterMetric("disk_used", startTime);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/timeline/PhoenixHBaseAccessorTest.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/timeline/PhoenixHBaseAccessorTest.java b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/timeline/PhoenixHBaseAccessorTest.java
new file mode 100644
index 0000000..8c31691
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/timeline/PhoenixHBaseAccessorTest.java
@@ -0,0 +1,291 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.metrics.core.timeline;
+
+import com.google.common.collect.ArrayListMultimap;
+import com.google.common.collect.Multimap;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.DoNotRetryIOException;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.metrics2.sink.timeline.MetricClusterAggregate;
+import org.apache.hadoop.metrics2.sink.timeline.MetricHostAggregate;
+import org.apache.hadoop.metrics2.sink.timeline.Precision;
+import org.apache.hadoop.metrics2.sink.timeline.TimelineMetric;
+import org.apache.hadoop.metrics2.sink.timeline.TimelineMetrics;
+import org.apache.ambari.metrics.core.timeline.aggregators.TimelineClusterMetric;
+import org.apache.ambari.metrics.core.timeline.aggregators.Function;
+import org.apache.ambari.metrics.core.timeline.discovery.TimelineMetricMetadataManager;
+import org.apache.ambari.metrics.core.timeline.query.Condition;
+import org.apache.ambari.metrics.core.timeline.query.DefaultCondition;
+import org.apache.ambari.metrics.core.timeline.query.PhoenixConnectionProvider;
+import org.apache.ambari.metrics.core.timeline.query.PhoenixTransactSQL;
+import org.apache.phoenix.exception.PhoenixIOException;
+import org.easymock.EasyMock;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.powermock.core.classloader.annotations.PrepareForTest;
+import org.powermock.modules.junit4.PowerMockRunner;
+
+import java.io.IOException;
+import java.sql.Connection;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+
+import static org.easymock.EasyMock.anyObject;
+import static org.easymock.EasyMock.expect;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+import static org.powermock.api.easymock.PowerMock.*;
+
+@RunWith(PowerMockRunner.class)
+@PrepareForTest({PhoenixTransactSQL.class, TimelineMetricConfiguration.class})
+public class PhoenixHBaseAccessorTest {
+ private static final String ZOOKEEPER_QUORUM = "hbase.zookeeper.quorum";
+
+ PhoenixConnectionProvider connectionProvider;
+ PhoenixHBaseAccessor accessor;
+
+ @Before
+ public void setupConf() throws Exception {
+ Configuration hbaseConf = new Configuration();
+ hbaseConf.setStrings(ZOOKEEPER_QUORUM, "quorum");
+ Configuration metricsConf = new Configuration();
+ metricsConf.setStrings(TimelineMetricConfiguration.TIMELINE_METRICS_CACHE_SIZE, "1");
+ metricsConf.setStrings(TimelineMetricConfiguration.TIMELINE_METRICS_CACHE_COMMIT_INTERVAL, "100");
+ metricsConf.setStrings(
+ TimelineMetricConfiguration.TIMELINE_METRIC_AGGREGATOR_SINK_CLASS,
+ "org.apache.ambari.metrics.core.timeline.TimelineMetricsAggregatorMemorySink");
+
+ TimelineMetricConfiguration conf = new TimelineMetricConfiguration(hbaseConf, metricsConf);
+ mockStatic(TimelineMetricConfiguration.class);
+ expect(TimelineMetricConfiguration.getInstance()).andReturn(conf).anyTimes();
+ replayAll();
+
+ connectionProvider = new PhoenixConnectionProvider() {
+ @Override
+ public HBaseAdmin getHBaseAdmin() throws IOException {
+ return null;
+ }
+
+ @Override
+ public Connection getConnection() throws SQLException {
+ return null;
+ }
+
+ };
+
+ accessor = new PhoenixHBaseAccessor(connectionProvider);
+ }
+
+ @Test
+ public void testGetMetricRecords() throws SQLException, IOException {
+ List<String> metricNames = new LinkedList<>();
+ List<String> hostnames = new LinkedList<>();
+ Multimap<String, List<Function>> metricFunctions = ArrayListMultimap.create();
+
+ mockStatic(PhoenixTransactSQL.class);
+ PreparedStatement preparedStatementMock = EasyMock.createNiceMock(PreparedStatement.class);
+ Condition condition = new DefaultCondition(metricNames, hostnames, "appid", "instanceid", 123L, 234L, Precision.SECONDS, 10, true);
+ expect(PhoenixTransactSQL.prepareGetMetricsSqlStmt(null, condition)).andReturn(preparedStatementMock).once();
+ ResultSet rsMock = EasyMock.createNiceMock(ResultSet.class);
+ expect(preparedStatementMock.executeQuery()).andReturn(rsMock);
+
+
+ replayAll();
+ EasyMock.replay(preparedStatementMock, rsMock);
+
+ // Check when startTime < endTime
+ TimelineMetrics tml = accessor.getMetricRecords(condition, metricFunctions);
+
+ // Check when startTime > endTime
+ Condition condition2 = new DefaultCondition(metricNames, hostnames, "appid", "instanceid", 234L, 123L, Precision.SECONDS, 10, true);
+ TimelineMetrics tml2 = accessor.getMetricRecords(condition2, metricFunctions);
+ assertEquals(0, tml2.getMetrics().size());
+
+ verifyAll();
+ EasyMock.verify(preparedStatementMock, rsMock);
+ }
+
+ @Test
+ public void testGetMetricRecordsIOException() throws SQLException, IOException {
+ List<String> metricNames = new LinkedList<>();
+ List<String> hostnames = new LinkedList<>();
+ Multimap<String, List<Function>> metricFunctions = ArrayListMultimap.create();
+
+ mockStatic(PhoenixTransactSQL.class);
+ PreparedStatement preparedStatementMock = EasyMock.createNiceMock(PreparedStatement.class);
+ Condition condition = new DefaultCondition(metricNames, hostnames, "appid", "instanceid", 123L, 234L, Precision.SECONDS, 10, true);
+ expect(PhoenixTransactSQL.prepareGetMetricsSqlStmt(null, condition)).andReturn(preparedStatementMock).once();
+ ResultSet rsMock = EasyMock.createNiceMock(ResultSet.class);
+ RuntimeException runtimeException = EasyMock.createNiceMock(RuntimeException.class);
+ IOException io = EasyMock.createNiceMock(IOException.class);
+ expect(preparedStatementMock.executeQuery()).andThrow(runtimeException);
+ expect(runtimeException.getCause()).andReturn(io).atLeastOnce();
+ StackTraceElement stackTrace[] = new StackTraceElement[]{new StackTraceElement("TimeRange","method","file",1)};
+ expect(io.getStackTrace()).andReturn(stackTrace).atLeastOnce();
+
+
+ replayAll();
+ EasyMock.replay(preparedStatementMock, rsMock, io, runtimeException);
+
+ TimelineMetrics tml = accessor.getMetricRecords(condition, metricFunctions);
+
+ assertEquals(0, tml.getMetrics().size());
+
+ verifyAll();
+ EasyMock.verify(preparedStatementMock, rsMock, io, runtimeException);
+ }
+
+ @Test
+ public void testGetMetricRecordsPhoenixIOExceptionDoNotRetryException() throws SQLException, IOException {
+ List<String> metricNames = new LinkedList<>();
+ List<String> hostnames = new LinkedList<>();
+ Multimap<String, List<Function>> metricFunctions = ArrayListMultimap.create();
+
+ mockStatic(PhoenixTransactSQL.class);
+ PreparedStatement preparedStatementMock = EasyMock.createNiceMock(PreparedStatement.class);
+ Condition condition = new DefaultCondition(metricNames, hostnames, "appid", "instanceid", null, null, Precision.SECONDS, 10, true);
+ expect(PhoenixTransactSQL.prepareGetLatestMetricSqlStmt(null, condition)).andReturn(preparedStatementMock).once();
+ PhoenixTransactSQL.setSortMergeJoinEnabled(true);
+ EasyMock.expectLastCall();
+ ResultSet rsMock = EasyMock.createNiceMock(ResultSet.class);
+ PhoenixIOException pioe1 = EasyMock.createNiceMock(PhoenixIOException.class);
+ PhoenixIOException pioe2 = EasyMock.createNiceMock(PhoenixIOException.class);
+ DoNotRetryIOException dnrioe = EasyMock.createNiceMock(DoNotRetryIOException.class);
+ expect(preparedStatementMock.executeQuery()).andThrow(pioe1);
+ expect(pioe1.getCause()).andReturn(pioe2).atLeastOnce();
+ expect(pioe2.getCause()).andReturn(dnrioe).atLeastOnce();
+ StackTraceElement stackTrace[] = new StackTraceElement[]{new StackTraceElement("HashJoinRegionScanner","method","file",1)};
+ expect(dnrioe.getStackTrace()).andReturn(stackTrace).atLeastOnce();
+
+
+ replayAll();
+ EasyMock.replay(preparedStatementMock, rsMock, pioe1, pioe2, dnrioe);
+ try {
+ accessor.getMetricRecords(condition, metricFunctions);
+ fail();
+ } catch (Exception e) {
+ //NOP
+ }
+ verifyAll();
+ }
+
+ @Test
+ public void testMetricsCacheCommittingWhenFull() throws IOException, SQLException {
+ Configuration hbaseConf = new Configuration();
+ hbaseConf.setStrings(ZOOKEEPER_QUORUM, "quorum");
+
+ final Connection connection = EasyMock.createNiceMock(Connection.class);
+
+ accessor = new PhoenixHBaseAccessor(connectionProvider) {
+ @Override
+ public void commitMetrics(Collection<TimelineMetrics> timelineMetricsCollection) {
+ try {
+ connection.commit();
+ } catch (SQLException e) {
+ e.printStackTrace();
+ }
+ }
+ };
+
+ TimelineMetrics timelineMetrics = EasyMock.createNiceMock(TimelineMetrics.class);
+ expect(timelineMetrics.getMetrics()).andReturn(Collections.singletonList(new TimelineMetric())).anyTimes();
+ connection.commit();
+ EasyMock.expectLastCall().once();
+
+ EasyMock.replay(timelineMetrics, connection);
+
+ accessor.insertMetricRecords(timelineMetrics);
+ accessor.insertMetricRecords(timelineMetrics);
+ accessor.insertMetricRecords(timelineMetrics);
+
+ EasyMock.verify(timelineMetrics, connection);
+ }
+
+ @Test
+ public void testMetricsAggregatorSink() throws IOException, SQLException {
+ Map<TimelineClusterMetric, MetricClusterAggregate> clusterAggregateMap =
+ new HashMap<>();
+ Map<TimelineClusterMetric, MetricHostAggregate> clusterTimeAggregateMap =
+ new HashMap<>();
+ Map<TimelineMetric, MetricHostAggregate> hostAggregateMap = new HashMap<>();
+
+
+ final Connection connection = EasyMock.createNiceMock(Connection.class);
+ final PreparedStatement statement = EasyMock.createNiceMock(PreparedStatement.class);
+ expect(connection.prepareStatement(EasyMock.anyString())).andReturn(statement).anyTimes();
+ EasyMock.replay(statement);
+ EasyMock.replay(connection);
+
+ connectionProvider = new PhoenixConnectionProvider() {
+
+ @Override
+ public HBaseAdmin getHBaseAdmin() throws IOException {
+ return null;
+ }
+
+ @Override
+ public Connection getConnection() throws SQLException {
+ return connection;
+ }
+ };
+
+ accessor = new PhoenixHBaseAccessor(connectionProvider);
+
+ TimelineClusterMetric clusterMetric =
+ new TimelineClusterMetric("metricName", "appId", "instanceId",
+ System.currentTimeMillis());
+ TimelineMetric timelineMetric = new TimelineMetric();
+ timelineMetric.setMetricName("Metric1");
+ timelineMetric.setType("type1");
+ timelineMetric.setAppId("App1");
+ timelineMetric.setInstanceId("instance1");
+ timelineMetric.setHostName("host1");
+
+ clusterAggregateMap.put(clusterMetric, new MetricClusterAggregate());
+ clusterTimeAggregateMap.put(clusterMetric, new MetricHostAggregate());
+ hostAggregateMap.put(timelineMetric, new MetricHostAggregate());
+
+ TimelineMetricMetadataManager metricMetadataManagerMock = EasyMock.createMock(TimelineMetricMetadataManager.class);
+ expect(metricMetadataManagerMock.getUuid(anyObject(TimelineClusterMetric.class))).andReturn(new byte[16]).times(2);
+ expect(metricMetadataManagerMock.getUuid(anyObject(TimelineMetric.class))).andReturn(new byte[20]).once();
+ replay(metricMetadataManagerMock);
+
+ accessor.setMetadataInstance(metricMetadataManagerMock);
+ accessor.saveClusterAggregateRecords(clusterAggregateMap);
+ accessor.saveHostAggregateRecords(hostAggregateMap,
+ PhoenixTransactSQL.METRICS_AGGREGATE_MINUTE_TABLE_NAME);
+ accessor.saveClusterAggregateRecordsSecond(clusterTimeAggregateMap,
+ PhoenixTransactSQL.METRICS_CLUSTER_AGGREGATE_HOURLY_TABLE_NAME);
+
+ TimelineMetricsAggregatorMemorySink memorySink =
+ new TimelineMetricsAggregatorMemorySink();
+ assertEquals(1, memorySink.getClusterAggregateRecords().size());
+ assertEquals(1, memorySink.getClusterTimeAggregateRecords().size());
+ assertEquals(1, memorySink.getHostAggregateRecords().size());
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/timeline/TestClusterSuite.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/timeline/TestClusterSuite.java b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/timeline/TestClusterSuite.java
new file mode 100644
index 0000000..b8c28ed
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/timeline/TestClusterSuite.java
@@ -0,0 +1,34 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.metrics.core.timeline;
+
+
+import org.apache.ambari.metrics.core.timeline.aggregators.ITClusterAggregator;
+import org.apache.ambari.metrics.core.timeline.aggregators.ITMetricAggregator;
+import org.junit.Ignore;
+import org.junit.runner.RunWith;
+import org.junit.runners.Suite;
+
+import static org.junit.runners.Suite.SuiteClasses;
+
+@Ignore
+@RunWith(Suite.class)
+@SuiteClasses({ITMetricAggregator.class, ITClusterAggregator.class, ITPhoenixHBaseAccessor.class})
+public class TestClusterSuite {
+
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/timeline/TestMetricHostAggregate.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/timeline/TestMetricHostAggregate.java b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/timeline/TestMetricHostAggregate.java
new file mode 100644
index 0000000..ad27b74
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/timeline/TestMetricHostAggregate.java
@@ -0,0 +1,65 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.metrics.core.timeline;
+
+import org.apache.hadoop.metrics2.sink.timeline.MetricHostAggregate;
+import org.junit.Test;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+public class TestMetricHostAggregate {
+
+ @Test
+ public void testCreateAggregate() throws Exception {
+ // given
+ MetricHostAggregate aggregate = createAggregate(3.0, 1.0, 2.0, 2);
+
+ //then
+ assertThat(aggregate.getSum()).isEqualTo(3.0);
+ assertThat(aggregate.getMin()).isEqualTo(1.0);
+ assertThat(aggregate.getMax()).isEqualTo(2.0);
+ assertThat(aggregate.calculateAverage()).isEqualTo(3.0 / 2);
+ }
+
+ @Test
+ public void testUpdateAggregates() throws Exception {
+ // given
+ MetricHostAggregate aggregate = createAggregate(3.0, 1.0, 2.0, 2);
+
+ //when
+ aggregate.updateAggregates(createAggregate(8.0, 0.5, 7.5, 2));
+ aggregate.updateAggregates(createAggregate(1.0, 1.0, 1.0, 1));
+
+ //then
+ assertThat(aggregate.getSum()).isEqualTo(12.0);
+ assertThat(aggregate.getMin()).isEqualTo(0.5);
+ assertThat(aggregate.getMax()).isEqualTo(7.5);
+ assertThat(aggregate.calculateAverage()).isEqualTo((3.0 + 8.0 + 1.0) / 5);
+ }
+
+ static MetricHostAggregate createAggregate (Double sum, Double min,
+ Double max, Integer samplesCount) {
+ MetricHostAggregate aggregate = new MetricHostAggregate();
+ aggregate.setSum(sum);
+ aggregate.setMax(max);
+ aggregate.setMin(min);
+ aggregate.setDeviation(0.0);
+ aggregate.setNumberOfSamples(samplesCount);
+ return aggregate;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/timeline/TestPhoenixTransactSQL.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/timeline/TestPhoenixTransactSQL.java b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/timeline/TestPhoenixTransactSQL.java
new file mode 100644
index 0000000..656034b
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/timeline/TestPhoenixTransactSQL.java
@@ -0,0 +1,608 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.metrics.core.timeline;
+
+import org.apache.hadoop.metrics2.sink.timeline.Precision;
+import org.apache.hadoop.metrics2.sink.timeline.PrecisionLimitExceededException;
+import org.apache.ambari.metrics.core.timeline.query.Condition;
+import org.apache.ambari.metrics.core.timeline.query.DefaultCondition;
+import org.apache.ambari.metrics.core.timeline.query.PhoenixTransactSQL;
+import org.apache.ambari.metrics.core.timeline.query.SplitByMetricNamesCondition;
+import org.apache.ambari.metrics.core.timeline.query.TopNCondition;
+import org.easymock.Capture;
+import org.junit.Assert;
+import org.junit.Ignore;
+import org.junit.Test;
+import java.sql.Connection;
+import java.sql.ParameterMetaData;
+import java.sql.PreparedStatement;
+import java.sql.SQLException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+
+import static org.easymock.EasyMock.createNiceMock;
+import static org.easymock.EasyMock.expect;
+import static org.easymock.EasyMock.replay;
+import static org.easymock.EasyMock.reset;
+import static org.easymock.EasyMock.verify;
+import org.easymock.EasyMock;
+
+public class TestPhoenixTransactSQL {
+ @Test
+ public void testConditionClause() throws Exception {
+ Condition condition = new DefaultCondition(Arrays.asList(new byte[8], new byte[8]),
+ new ArrayList<>(Arrays.asList("cpu_user", "mem_free")), Collections.singletonList("h1"),
+ "a1", "i1", 1407959718L, 1407959918L, null, null, false);
+
+ String preparedClause = condition.getConditionClause().toString();
+ String expectedClause = "(UUID IN (?, ?)) AND SERVER_TIME >= ? AND SERVER_TIME < ?";
+
+ Assert.assertNotNull(preparedClause);
+ Assert.assertEquals(expectedClause, preparedClause);
+ }
+
+ @Test
+ public void testSplitByMetricNamesCondition() throws Exception {
+ Condition c = new DefaultCondition(Arrays.asList(new byte[8], new byte[8]),
+ Arrays.asList("cpu_user", "mem_free"), Collections.singletonList("h1"),
+ "a1", "i1", 1407959718L, 1407959918L, null, null, false);
+
+ SplitByMetricNamesCondition condition = new SplitByMetricNamesCondition(c);
+ condition.setCurrentUuid(new byte[8]);
+
+ String preparedClause = condition.getConditionClause().toString();
+ String expectedClause = "UUID = ? AND SERVER_TIME >= ? AND SERVER_TIME < ?";
+
+ Assert.assertNotNull(preparedClause);
+ Assert.assertEquals(expectedClause, preparedClause);
+ }
+
+ @Ignore
+ @Test
+ public void testLikeConditionClause() throws Exception {
+ Condition condition = new DefaultCondition(
+ new ArrayList<>(Arrays.asList("cpu_user", "some=%.metric")),
+ Collections.singletonList("h1"), "a1", "i1", 1407959718L, 1407959918L,
+ null, null, false);
+
+ String preparedClause = condition.getConditionClause().toString();
+ String expectedClause = "(METRIC_NAME IN (?) OR METRIC_NAME LIKE ?) AND HOSTNAME = ? AND " +
+ "APP_ID = ? AND INSTANCE_ID = ? AND SERVER_TIME >= ? AND SERVER_TIME < ?";
+
+ Assert.assertNotNull(preparedClause);
+ Assert.assertEquals(expectedClause, preparedClause);
+
+
+ condition = new DefaultCondition(
+ Collections.<String>emptyList(), Collections.singletonList("h1"), "a1", "i1",
+ 1407959718L, 1407959918L, null, null, false);
+
+ preparedClause = condition.getConditionClause().toString();
+ expectedClause = " HOSTNAME = ? AND " +
+ "APP_ID = ? AND INSTANCE_ID = ? AND SERVER_TIME >= ? AND SERVER_TIME < ?";
+
+ Assert.assertNotNull(preparedClause);
+ Assert.assertEquals(expectedClause, preparedClause);
+
+
+ condition = new DefaultCondition(
+ null, Collections.singletonList("h1"), "a1", "i1",
+ 1407959718L, 1407959918L, null, null, false);
+
+ preparedClause = condition.getConditionClause().toString();
+ expectedClause = " HOSTNAME = ? AND " +
+ "APP_ID = ? AND INSTANCE_ID = ? AND SERVER_TIME >= ? AND SERVER_TIME < ?";
+
+ Assert.assertNotNull(preparedClause);
+ Assert.assertEquals(expectedClause, preparedClause);
+
+
+ condition = new DefaultCondition(
+ new ArrayList<>(Arrays.asList("some=%.metric")), Collections.singletonList("h1"), "a1", "i1",
+ 1407959718L, 1407959918L, null, null, false);
+
+ preparedClause = condition.getConditionClause().toString();
+ expectedClause = "(METRIC_NAME LIKE ?) AND HOSTNAME = ? AND " +
+ "APP_ID = ? AND INSTANCE_ID = ? AND SERVER_TIME >= ? AND SERVER_TIME < ?";
+
+ Assert.assertNotNull(preparedClause);
+ Assert.assertEquals(expectedClause, preparedClause);
+
+
+ condition = new DefaultCondition(
+ new ArrayList<>(Arrays.asList("some=%.metric1", "some=%.metric2", "some=%.metric3")),
+ Collections.singletonList("h1"), "a1", "i1",
+ 1407959718L, 1407959918L, null, null, false);
+
+ preparedClause = condition.getConditionClause().toString();
+ expectedClause = "(METRIC_NAME LIKE ? OR METRIC_NAME LIKE ? OR METRIC_NAME LIKE ?) AND HOSTNAME = ? AND " +
+ "APP_ID = ? AND INSTANCE_ID = ? AND SERVER_TIME >= ? AND SERVER_TIME < ?";
+
+ Assert.assertNotNull(preparedClause);
+ Assert.assertEquals(expectedClause, preparedClause);
+ }
+
+ @Test
+ public void testPrepareGetAggregatePrecisionMINUTES() throws SQLException {
+ Condition condition = new DefaultCondition(
+ new ArrayList<>(Arrays.asList("cpu_user", "mem_free")), Collections.singletonList("h1"),
+ "a1", "i1", 1407959718L, 1407959918L, Precision.MINUTES, null, false);
+ Connection connection = createNiceMock(Connection.class);
+ PreparedStatement preparedStatement = createNiceMock(PreparedStatement.class);
+ Capture<String> stmtCapture = new Capture<String>();
+ expect(connection.prepareStatement(EasyMock.and(EasyMock.anyString(), EasyMock.capture(stmtCapture))))
+ .andReturn(preparedStatement);
+
+ replay(connection, preparedStatement);
+ PhoenixTransactSQL.prepareGetAggregateSqlStmt(connection, condition);
+ String stmt = stmtCapture.getValue();
+ Assert.assertTrue(stmt.contains("FROM METRIC_AGGREGATE"));
+ verify(connection, preparedStatement);
+ }
+
+ @Test
+ public void testPrepareGetAggregateNoPrecision() throws SQLException {
+ Long endTime = 1407959918L;
+ Long startTime = 1407959718L;
+ //SECONDS precision
+ Condition condition = new DefaultCondition(
+ new ArrayList<>(Arrays.asList("cpu_user", "mem_free")), Collections.singletonList("h1"),
+ "a1", "i1", startTime, endTime, null, null, false);
+ Connection connection = createNiceMock(Connection.class);
+ PreparedStatement preparedStatement = createNiceMock(PreparedStatement.class);
+ Capture<String> stmtCapture = new Capture<String>();
+ expect(connection.prepareStatement(EasyMock.and(EasyMock.anyString(), EasyMock.capture(stmtCapture))))
+ .andReturn(preparedStatement);
+
+ replay(connection, preparedStatement);
+ PhoenixTransactSQL.prepareGetAggregateSqlStmt(connection, condition);
+ String stmt = stmtCapture.getValue();
+ Assert.assertTrue(stmt.contains("FROM METRIC_AGGREGATE"));
+ Assert.assertEquals(Precision.SECONDS, condition.getPrecision());
+ verify(connection, preparedStatement);
+
+ // MINUTES precision
+ startTime = endTime-PhoenixTransactSQL.DAY/1000;
+ condition = new DefaultCondition(
+ new ArrayList<>(Arrays.asList("cpu_user", "mem_free")), Collections.singletonList("h1"),
+ "a1", "i1", startTime, endTime, null, null, false);
+ connection = createNiceMock(Connection.class);
+ preparedStatement = createNiceMock(PreparedStatement.class);
+ stmtCapture = new Capture<String>();
+ expect(connection.prepareStatement(EasyMock.and(EasyMock.anyString(), EasyMock.capture(stmtCapture))))
+ .andReturn(preparedStatement);
+
+ replay(connection, preparedStatement);
+ PhoenixTransactSQL.prepareGetAggregateSqlStmt(connection, condition);
+ stmt = stmtCapture.getValue();
+ Assert.assertTrue(stmt.contains("FROM METRIC_AGGREGATE"));
+ Assert.assertEquals(Precision.MINUTES, condition.getPrecision());
+ verify(connection, preparedStatement);
+
+ // HOURS precision
+ startTime = endTime-PhoenixTransactSQL.DAY*30/1000;
+ condition = new DefaultCondition(
+ new ArrayList<>(Arrays.asList("cpu_user", "mem_free")), Collections.singletonList("h1"),
+ "a1", "i1", startTime, endTime, null, null, false);
+ connection = createNiceMock(Connection.class);
+ preparedStatement = createNiceMock(PreparedStatement.class);
+ stmtCapture = new Capture<String>();
+ expect(connection.prepareStatement(EasyMock.and(EasyMock.anyString(), EasyMock.capture(stmtCapture))))
+ .andReturn(preparedStatement);
+
+ replay(connection, preparedStatement);
+ PhoenixTransactSQL.prepareGetAggregateSqlStmt(connection, condition);
+ stmt = stmtCapture.getValue();
+ Assert.assertTrue(stmt.contains("FROM METRIC_AGGREGATE_HOURLY"));
+ Assert.assertEquals(Precision.HOURS, condition.getPrecision());
+ verify(connection, preparedStatement);
+
+ // DAYS precision
+ startTime = endTime-PhoenixTransactSQL.DAY*30*2/1000;
+ condition = new DefaultCondition(
+ new ArrayList<>(Arrays.asList("cpu_user", "mem_free")), Collections.singletonList("h1"),
+ "a1", "i1", startTime, endTime, null, null, false);
+ connection = createNiceMock(Connection.class);
+ preparedStatement = createNiceMock(PreparedStatement.class);
+ stmtCapture = new Capture<String>();
+ expect(connection.prepareStatement(EasyMock.and(EasyMock.anyString(), EasyMock.capture(stmtCapture))))
+ .andReturn(preparedStatement);
+
+ replay(connection, preparedStatement);
+ PhoenixTransactSQL.prepareGetAggregateSqlStmt(connection, condition);
+ stmt = stmtCapture.getValue();
+ Assert.assertTrue(stmt.contains("FROM METRIC_AGGREGATE_DAILY"));
+ Assert.assertEquals(Precision.DAYS, condition.getPrecision());
+ verify(connection, preparedStatement);
+ }
+
+ @Test
+ public void testPrepareGetAggregatePrecisionHours() throws SQLException {
+ Condition condition = new DefaultCondition(
+ new ArrayList<>(Arrays.asList("cpu_user", "mem_free")), Collections.singletonList("h1"),
+ "a1", "i1", 1407959718L, 1407959918L, Precision.HOURS, null, false);
+ Connection connection = createNiceMock(Connection.class);
+ PreparedStatement preparedStatement = createNiceMock(PreparedStatement.class);
+ Capture<String> stmtCapture = new Capture<String>();
+ expect(connection.prepareStatement(EasyMock.and(EasyMock.anyString(), EasyMock.capture(stmtCapture))))
+ .andReturn(preparedStatement);
+
+ replay(connection, preparedStatement);
+ PhoenixTransactSQL.prepareGetAggregateSqlStmt(connection, condition);
+ String stmt = stmtCapture.getValue();
+ Assert.assertTrue(stmt.contains("FROM METRIC_AGGREGATE_HOURLY"));
+ verify(connection, preparedStatement);
+ }
+
+ @Test
+ public void testPrepareGetMetricsPrecisionMinutes() throws SQLException {
+ Condition condition = new DefaultCondition(
+ new ArrayList<>(Arrays.asList("cpu_user", "mem_free")), Collections.singletonList("h1"),
+ "a1", "i1", 1407959718L, 1407959918L, Precision.MINUTES, null, false);
+ Connection connection = createNiceMock(Connection.class);
+ PreparedStatement preparedStatement = createNiceMock(PreparedStatement.class);
+ Capture<String> stmtCapture = new Capture<String>();
+ expect(connection.prepareStatement(EasyMock.and(EasyMock.anyString(), EasyMock.capture(stmtCapture))))
+ .andReturn(preparedStatement);
+
+ replay(connection, preparedStatement);
+ PhoenixTransactSQL.prepareGetMetricsSqlStmt(connection, condition);
+ String stmt = stmtCapture.getValue();
+ Assert.assertTrue(stmt.contains("FROM METRIC_RECORD_MINUTE"));
+ verify(connection, preparedStatement);
+ }
+
+ @Test
+ public void testPrepareGetMetricsNoPrecision() throws SQLException {
+ Long endTime = 1407959918L;
+ Long startTime = endTime - 200;
+ // SECONDS precision
+ Condition condition = new DefaultCondition(
+ new ArrayList<>(Arrays.asList("cpu_user", "mem_free")), Collections.singletonList("h1"),
+ "a1", "i1", startTime, endTime, null, null, false);
+ Connection connection = createNiceMock(Connection.class);
+ PreparedStatement preparedStatement = createNiceMock(PreparedStatement.class);
+ Capture<String> stmtCapture = new Capture<String>();
+ expect(connection.prepareStatement(EasyMock.and(EasyMock.anyString(), EasyMock.capture(stmtCapture))))
+ .andReturn(preparedStatement);
+
+ replay(connection, preparedStatement);
+ PhoenixTransactSQL.prepareGetMetricsSqlStmt(connection, condition);
+ String stmt = stmtCapture.getValue();
+ Assert.assertTrue(stmt.contains("FROM METRIC_RECORD"));
+ Assert.assertEquals(Precision.SECONDS, condition.getPrecision());
+ verify(connection, preparedStatement);
+ reset(connection, preparedStatement);
+
+ // SECONDS precision
+ startTime = endTime-PhoenixTransactSQL.HOUR*2/1000;
+ condition = new DefaultCondition(
+ new ArrayList<>(Arrays.asList("cpu_user", "mem_free")), Collections.singletonList("h1"),
+ "a1", "i1", startTime, endTime, null, null, false);
+ connection = createNiceMock(Connection.class);
+ preparedStatement = createNiceMock(PreparedStatement.class);
+ stmtCapture = new Capture<String>();
+ expect(connection.prepareStatement(EasyMock.and(EasyMock.anyString(), EasyMock.capture(stmtCapture))))
+ .andReturn(preparedStatement);
+ replay(connection, preparedStatement);
+ PhoenixTransactSQL.prepareGetMetricsSqlStmt(connection, condition);
+ stmt = stmtCapture.getValue();
+ Assert.assertTrue(stmt.contains("FROM METRIC_RECORD"));
+ Assert.assertEquals(Precision.SECONDS, condition.getPrecision());
+ verify(connection, preparedStatement);
+
+ // MINUTES precision
+ startTime = endTime-PhoenixTransactSQL.DAY/1000;
+ condition = new DefaultCondition(
+ new ArrayList<>(Arrays.asList("cpu_user", "mem_free")), Collections.singletonList("h1"),
+ "a1", "i1", startTime, endTime, null, null, false);
+ connection = createNiceMock(Connection.class);
+ preparedStatement = createNiceMock(PreparedStatement.class);
+ stmtCapture = new Capture<String>();
+ expect(connection.prepareStatement(EasyMock.and(EasyMock.anyString(), EasyMock.capture(stmtCapture))))
+ .andReturn(preparedStatement);
+ replay(connection, preparedStatement);
+ PhoenixTransactSQL.prepareGetMetricsSqlStmt(connection, condition);
+ stmt = stmtCapture.getValue();
+ Assert.assertTrue(stmt.contains("FROM METRIC_RECORD_MINUTE"));
+ Assert.assertEquals(Precision.MINUTES, condition.getPrecision());
+ verify(connection, preparedStatement);
+
+ // HOURS precision
+ startTime = endTime-PhoenixTransactSQL.DAY*30/1000;
+ condition = new DefaultCondition(
+ new ArrayList<>(Arrays.asList("cpu_user", "mem_free")), Collections.singletonList("h1"),
+ "a1", "i1", startTime, endTime, null, null, false);
+ connection = createNiceMock(Connection.class);
+ preparedStatement = createNiceMock(PreparedStatement.class);
+ stmtCapture = new Capture<String>();
+ expect(connection.prepareStatement(EasyMock.and(EasyMock.anyString(), EasyMock.capture(stmtCapture))))
+ .andReturn(preparedStatement);
+ replay(connection, preparedStatement);
+ PhoenixTransactSQL.prepareGetMetricsSqlStmt(connection, condition);
+ stmt = stmtCapture.getValue();
+ Assert.assertTrue(stmt.contains("FROM METRIC_RECORD_HOURLY"));
+ Assert.assertEquals(Precision.HOURS, condition.getPrecision());
+ verify(connection, preparedStatement);
+
+ // DAYS precision
+ startTime = endTime-PhoenixTransactSQL.DAY*30*2/1000;
+ condition = new DefaultCondition(
+ new ArrayList<>(Arrays.asList("cpu_user", "mem_free")), Collections.singletonList("h1"),
+ "a1", "i1", startTime, endTime, null, null, false);
+ connection = createNiceMock(Connection.class);
+ preparedStatement = createNiceMock(PreparedStatement.class);
+ stmtCapture = new Capture<String>();
+ expect(connection.prepareStatement(EasyMock.and(EasyMock.anyString(), EasyMock.capture(stmtCapture))))
+ .andReturn(preparedStatement);
+ replay(connection, preparedStatement);
+ PhoenixTransactSQL.prepareGetMetricsSqlStmt(connection, condition);
+ stmt = stmtCapture.getValue();
+ Assert.assertTrue(stmt.contains("FROM METRIC_RECORD_DAILY"));
+ Assert.assertEquals(Precision.DAYS, condition.getPrecision());
+ verify(connection, preparedStatement);
+
+ }
+
+ @Test
+ public void testPrepareGetLatestMetricSqlStmtMultipleHostNames() throws SQLException {
+ Condition condition = new DefaultCondition(Arrays.asList(new byte[16], new byte[16], new byte[16], new byte[16]),
+ new ArrayList<>(Arrays.asList("cpu_user", "mem_free")), Arrays.asList("h1", "h2"),
+ "a1", "i1", null, null, null, null, false);
+ Connection connection = createNiceMock(Connection.class);
+ PreparedStatement preparedStatement = createNiceMock(PreparedStatement.class);
+ ParameterMetaData parameterMetaData = createNiceMock(ParameterMetaData.class);
+ Capture<String> stmtCapture = new Capture<String>();
+ expect(connection.prepareStatement(EasyMock.and(EasyMock.anyString(), EasyMock.capture(stmtCapture))))
+ .andReturn(preparedStatement);
+ expect(preparedStatement.getParameterMetaData())
+ .andReturn(parameterMetaData).once();
+ // 6 = 1 instance_id + 1 appd_id + 2 hostnames + 2 metric names
+ expect(parameterMetaData.getParameterCount())
+ .andReturn(4).once();
+
+ replay(connection, preparedStatement, parameterMetaData);
+ PhoenixTransactSQL.prepareGetLatestMetricSqlStmt(connection, condition);
+ String stmt = stmtCapture.getValue();
+ Assert.assertTrue(stmt.contains("FROM METRIC_RECORD"));
+ Assert.assertTrue(stmt.contains("JOIN"));
+ verify(connection, preparedStatement, parameterMetaData);
+ }
+
+ @Test
+ public void testPrepareGetLatestMetricSqlStmtSortMergeJoinAlgorithm()
+ throws SQLException {
+ Condition condition = new DefaultCondition(Arrays.asList(new byte[16], new byte[16]),
+ new ArrayList<>(Arrays.asList("cpu_user", "mem_free")), Arrays.asList("h1"),
+ "a1", "i1", null, null, null, null, false);
+ Connection connection = createNiceMock(Connection.class);
+ PreparedStatement preparedStatement = createNiceMock(PreparedStatement.class);
+ ParameterMetaData parameterMetaData = createNiceMock(ParameterMetaData.class);
+ Capture<String> stmtCapture = new Capture<String>();
+ expect(connection.prepareStatement(EasyMock.and(EasyMock.anyString(), EasyMock.capture(stmtCapture))))
+ .andReturn(preparedStatement);
+ expect(preparedStatement.getParameterMetaData())
+ .andReturn(parameterMetaData).anyTimes();
+ expect(parameterMetaData.getParameterCount())
+ .andReturn(2).anyTimes();
+
+ replay(connection, preparedStatement, parameterMetaData);
+ PhoenixTransactSQL.setSortMergeJoinEnabled(true);
+ PhoenixTransactSQL.prepareGetLatestMetricSqlStmt(connection, condition);
+ String stmt = stmtCapture.getValue();
+ Assert.assertTrue(stmt.contains("/*+ USE_SORT_MERGE_JOIN NO_CACHE */"));
+ }
+
+ @Test
+ public void testPrepareGetMetricsPrecisionHours() throws SQLException {
+ Condition condition = new DefaultCondition(
+ new ArrayList<>(Arrays.asList("cpu_user", "mem_free")), Collections.singletonList("h1"),
+ "a1", "i1", 1407959718L, 1407959918L, Precision.HOURS, null, false);
+ Connection connection = createNiceMock(Connection.class);
+ PreparedStatement preparedStatement = createNiceMock(PreparedStatement.class);
+ Capture<String> stmtCapture = new Capture<String>();
+ expect(connection.prepareStatement(EasyMock.and(EasyMock.anyString(), EasyMock.capture(stmtCapture))))
+ .andReturn(preparedStatement);
+
+ replay(connection, preparedStatement);
+ PhoenixTransactSQL.prepareGetMetricsSqlStmt(connection, condition);
+ String stmt = stmtCapture.getValue();
+ Assert.assertTrue(stmt.contains("FROM METRIC_RECORD_HOURLY"));
+ verify(connection, preparedStatement);
+ }
+
+ @Test
+ public void testResultSetLimitCheck() throws SQLException {
+
+ List<String> metrics = new ArrayList<String>();
+ List<String> hosts = new ArrayList<String>();
+ int numMetrics = 0;
+ int numHosts = 0;
+ int limit = PhoenixHBaseAccessor.RESULTSET_LIMIT;
+
+ // 22 Metrics x 2 Hosts x 1 hour with requested SECONDS precision = 15840 points. Should be OK!
+ numMetrics = 22;
+ numHosts = 2;
+ for (int i = 0; i < numMetrics; i++) {
+ metrics.add("TestMetric"+i);
+ }
+ for (int i = 0; i < numHosts; i++) {
+ hosts.add("TestHost"+i);
+ }
+
+ Condition condition = new DefaultCondition(
+ metrics, hosts,
+ "a1", "i1", 1407950000L, 1407953600L, Precision.SECONDS, null, false);
+ Connection connection = createNiceMock(Connection.class);
+ PreparedStatement preparedStatement = createNiceMock(PreparedStatement.class);
+ Capture<String> stmtCapture = new Capture<String>();
+ expect(connection.prepareStatement(EasyMock.and(EasyMock.anyString(), EasyMock.capture(stmtCapture))))
+ .andReturn(preparedStatement);
+
+ replay(connection, preparedStatement);
+ PhoenixTransactSQL.prepareGetMetricsSqlStmt(connection, condition);
+ String stmt = stmtCapture.getValue();
+ Assert.assertTrue(stmt.contains("FROM METRIC_RECORD"));
+ verify(connection, preparedStatement);
+
+ //Check without passing precision. Should be OK!
+ condition = new DefaultCondition(
+ metrics, hosts,
+ "a1", "i1", 1407950000L, 1407953600L, null, null, false);
+ connection = createNiceMock(Connection.class);
+ preparedStatement = createNiceMock(PreparedStatement.class);
+ stmtCapture = new Capture<String>();
+ expect(connection.prepareStatement(EasyMock.and(EasyMock.anyString(), EasyMock.capture(stmtCapture))))
+ .andReturn(preparedStatement);
+
+ replay(connection, preparedStatement);
+ PhoenixTransactSQL.prepareGetMetricsSqlStmt(connection, condition);
+ stmt = stmtCapture.getValue();
+ Assert.assertTrue(stmt.contains("FROM METRIC_RECORD"));
+ verify(connection, preparedStatement);
+
+ //Check with more hosts and lesser metrics for 1 day data = 11520 points Should be OK!
+ metrics.clear();
+ hosts.clear();
+ numMetrics = 2;
+ numHosts = 20;
+ for (int i = 0; i < numMetrics; i++) {
+ metrics.add("TestMetric"+i);
+ }
+ for (int i = 0; i < numHosts; i++) {
+ hosts.add("TestHost"+i);
+ }
+ condition = new DefaultCondition(
+ metrics, hosts,
+ "a1", "i1", 1407867200L, 1407953600L, null, null, false);
+ connection = createNiceMock(Connection.class);
+ preparedStatement = createNiceMock(PreparedStatement.class);
+ stmtCapture = new Capture<String>();
+ expect(connection.prepareStatement(EasyMock.and(EasyMock.anyString(), EasyMock.capture(stmtCapture))))
+ .andReturn(preparedStatement);
+
+ replay(connection, preparedStatement);
+ PhoenixTransactSQL.prepareGetMetricsSqlStmt(connection, condition);
+ stmt = stmtCapture.getValue();
+ Assert.assertTrue(stmt.contains("FROM METRIC_RECORD_MINUTE"));
+ verify(connection, preparedStatement);
+
+ //Check with 20 metrics, NO hosts and 1 day data = 5760 points. Should be OK!
+ metrics.clear();
+ hosts.clear();
+ numMetrics = 20;
+ for (int i = 0; i < numMetrics; i++) {
+ metrics.add("TestMetric"+i);
+ }
+ condition = new DefaultCondition(
+ metrics, hosts,
+ "a1", "i1", 1407867200L, 1407953600L, null, null, false);
+ connection = createNiceMock(Connection.class);
+ preparedStatement = createNiceMock(PreparedStatement.class);
+ stmtCapture = new Capture<String>();
+ expect(connection.prepareStatement(EasyMock.and(EasyMock.anyString(), EasyMock.capture(stmtCapture))))
+ .andReturn(preparedStatement);
+
+ replay(connection, preparedStatement);
+ PhoenixTransactSQL.prepareGetAggregateSqlStmt(connection, condition);
+ stmt = stmtCapture.getValue();
+ Assert.assertTrue(stmt.contains("FROM METRIC_AGGREGATE_MINUTE"));
+ verify(connection, preparedStatement);
+
+ //Check with 5 hosts and 10 metrics for 1 hour data = 18000 points. Should throw out Exception!
+ metrics.clear();
+ hosts.clear();
+ numMetrics = 10;
+ numHosts = 5;
+ for (int i = 0; i < numMetrics; i++) {
+ metrics.add("TestMetric"+i);
+ }
+ for (int i = 0; i < numHosts; i++) {
+ hosts.add("TestHost"+i);
+ }
+ condition = new DefaultCondition(
+ metrics, hosts,
+ "a1", "i1", 1407950000L, 1407953600L, null, null, false);
+ boolean exceptionThrown = false;
+ boolean requestedSizeFoundInMessage = false;
+
+ try {
+ PhoenixTransactSQL.prepareGetMetricsSqlStmt(connection, condition);
+ } catch (PrecisionLimitExceededException pe) {
+ exceptionThrown = true;
+ String message = pe.getMessage();
+ if (message !=null && message.contains("18000")) {
+ requestedSizeFoundInMessage = true;
+ }
+ }
+ Assert.assertTrue(exceptionThrown);
+ Assert.assertTrue(requestedSizeFoundInMessage);
+ }
+
+ @Test
+ public void testTopNHostsConditionClause() throws Exception {
+ List<String> hosts = Arrays.asList("h1", "h2");
+ List<byte[]> uuids = Arrays.asList(new byte[16], new byte[16]);
+
+ Condition condition = new TopNCondition(uuids, new ArrayList<>(Collections.singletonList("cpu_user")), hosts,
+ "a1", "i1", 1407959718L, 1407959918L, null, null, false, 2, null, false);
+
+ String conditionClause = condition.getConditionClause().toString();
+ String expectedClause = " UUID IN (" +
+ "SELECT UUID FROM METRIC_RECORD WHERE " +
+ "(UUID LIKE ? OR UUID LIKE ?) AND " +
+ "SERVER_TIME >= ? AND SERVER_TIME < ? " +
+ "GROUP BY UUID ORDER BY MAX(METRIC_MAX) DESC LIMIT 2) AND SERVER_TIME >= ? AND SERVER_TIME < ?";
+
+ Assert.assertEquals(expectedClause, conditionClause);
+ }
+
+ @Test
+ public void testTopNMetricsConditionClause() throws Exception {
+ List<String> metricNames = new ArrayList<>(Arrays.asList("m1", "m2", "m3"));
+ List<byte[]> uuids = Arrays.asList(new byte[16], new byte[16], new byte[16]);
+
+ Condition condition = new TopNCondition(uuids, metricNames, Collections.singletonList("h1"),
+ "a1", "i1", 1407959718L, 1407959918L, null, null, false, 2, null, false);
+
+ String conditionClause = condition.getConditionClause().toString();
+ String expectedClause = " UUID IN (" +
+ "SELECT UUID FROM METRIC_RECORD WHERE " +
+ "(UUID LIKE ? OR UUID LIKE ? OR UUID LIKE ?) AND " +
+ "SERVER_TIME >= ? AND SERVER_TIME < ? " +
+ "GROUP BY UUID ORDER BY MAX(METRIC_MAX) DESC LIMIT 2) AND SERVER_TIME >= ? AND SERVER_TIME < ?";
+
+ Assert.assertEquals(expectedClause, conditionClause);
+ }
+
+ @Test
+ public void testTopNMetricsIllegalConditionClause() throws Exception {
+ List<String> metricNames = new ArrayList<>(Arrays.asList("m1", "m2"));
+
+ List<String> hosts = Arrays.asList("h1", "h2");
+ List<byte[]> uuids = Arrays.asList(new byte[16], new byte[16], new byte[16], new byte[16]);
+
+ Condition condition = new TopNCondition(uuids, metricNames, hosts,
+ "a1", "i1", 1407959718L, 1407959918L, null, null, false, 2, null, false);
+
+ Assert.assertEquals(condition.getConditionClause(), null);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/timeline/TestTimelineMetricStore.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/timeline/TestTimelineMetricStore.java b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/timeline/TestTimelineMetricStore.java
new file mode 100644
index 0000000..99e3e25
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/timeline/TestTimelineMetricStore.java
@@ -0,0 +1,123 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.metrics.core.timeline;
+
+import java.io.IOException;
+import java.sql.SQLException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeMap;
+
+import org.apache.hadoop.metrics2.sink.timeline.AggregationResult;
+import org.apache.hadoop.metrics2.sink.timeline.ContainerMetric;
+import org.apache.hadoop.metrics2.sink.timeline.Precision;
+import org.apache.hadoop.metrics2.sink.timeline.TimelineMetric;
+import org.apache.hadoop.metrics2.sink.timeline.TimelineMetricMetadata;
+import org.apache.hadoop.metrics2.sink.timeline.TimelineMetrics;
+import org.apache.hadoop.metrics2.sink.timeline.TopNConfig;
+import org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse;
+
+public class TestTimelineMetricStore implements TimelineMetricStore {
+ @Override
+ public TimelineMetrics getTimelineMetrics(List<String> metricNames,
+ List<String> hostnames, String applicationId, String instanceId, Long startTime,
+ Long endTime, Precision precision, Integer limit, boolean groupedByHost,
+ TopNConfig topNConfig, String seriesAggregateFunction) throws SQLException,
+ IOException {
+ TimelineMetrics timelineMetrics = new TimelineMetrics();
+ List<TimelineMetric> metricList = new ArrayList<TimelineMetric>();
+ timelineMetrics.setMetrics(metricList);
+ TimelineMetric metric1 = new TimelineMetric();
+ TimelineMetric metric2 = new TimelineMetric();
+ metricList.add(metric1);
+ metricList.add(metric2);
+ metric1.setMetricName("cpu_user");
+ metric1.setAppId("1");
+ metric1.setInstanceId(null);
+ metric1.setHostName("c6401");
+ metric1.setStartTime(1407949812L);
+ metric1.setMetricValues(new TreeMap<Long, Double>() {{
+ put(1407949812L, 1.0d);
+ put(1407949912L, 1.8d);
+ put(1407950002L, 0.7d);
+ }});
+
+ metric2.setMetricName("mem_free");
+ metric2.setAppId("2");
+ metric2.setInstanceId("3");
+ metric2.setHostName("c6401");
+ metric2.setStartTime(1407949812L);
+ metric2.setMetricValues(new TreeMap<Long, Double>() {{
+ put(1407949812L, 2.5d);
+ put(1407949912L, 3.0d);
+ put(1407950002L, 0.9d);
+ }});
+
+ return timelineMetrics;
+ }
+
+ @Override
+ public TimelinePutResponse putMetrics(TimelineMetrics metrics)
+ throws SQLException, IOException {
+
+ return new TimelinePutResponse();
+ }
+
+ @Override
+ public TimelinePutResponse putContainerMetrics(List<ContainerMetric> metrics)
+ throws SQLException, IOException {
+ return new TimelinePutResponse();
+ }
+
+ @Override
+ public Map<String, List<TimelineMetricMetadata>> getTimelineMetricMetadata(String appId, String metricPattern,
+ boolean includeBlacklistedMetrics) throws SQLException, IOException {
+ return null;
+ }
+
+ @Override
+ public TimelinePutResponse putHostAggregatedMetrics(AggregationResult aggregationResult) throws SQLException, IOException {
+ return null;
+ }
+
+ @Override
+ public Map<String, Set<String>> getHostAppsMetadata() throws SQLException, IOException {
+ return Collections.emptyMap();
+ }
+
+ @Override
+ public Map<String, Map<String,Set<String>>> getInstanceHostsMetadata(String instanceId, String appId) throws SQLException, IOException {
+ return Collections.emptyMap();
+ }
+
+ @Override
+ public List<String> getLiveInstances() {
+ return Collections.emptyList();
+ }
+
+ @Override
+ public byte[] getUuid(String metricName, String appId, String instanceId, String hostname) throws SQLException, IOException {
+ return null;
+ }
+
+}
+
+
[05/28] ambari git commit: AMBARI-22740 : Fix integration test for
HBase in branch-3.0-ams due to UUID changes. (avijayan)
Posted by av...@apache.org.
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/timeline/uuid/TimelineMetricUuidManagerTest.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/timeline/uuid/TimelineMetricUuidManagerTest.java b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/timeline/uuid/TimelineMetricUuidManagerTest.java
new file mode 100644
index 0000000..65db9c8
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/timeline/uuid/TimelineMetricUuidManagerTest.java
@@ -0,0 +1,175 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.metrics.core.timeline.uuid;
+
+import org.apache.ambari.metrics.core.timeline.aggregators.TimelineClusterMetric;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.io.BufferedReader;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.net.URL;
+import java.sql.SQLException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+public class TimelineMetricUuidManagerTest {
+
+
+ private List<String> apps = Arrays.asList("namenode",
+ "datanode", "master_hbase", "slave_hbase", "kafka_broker", "nimbus", "ams-hbase",
+ "accumulo", "nodemanager", "resourcemanager", "ambari_server", "HOST", "timeline_metric_store_watcher",
+ "jobhistoryserver", "hiveserver2", "hivemetastore", "applicationhistoryserver", "amssmoketestfake");
+
+ private Map<String, Set<String>> metricSet = new HashMap<>(populateMetricWhitelistFromFile());
+
+ @Test
+ public void testHashBasedUuidForMetricName() throws SQLException {
+
+ MetricUuidGenStrategy strategy = new HashBasedUuidGenStrategy();
+ Map<String, TimelineClusterMetric> uuids = new HashMap<>();
+ for (String app : metricSet.keySet()) {
+ Set<String> metrics = metricSet.get(app);
+ for (String metric : metrics) {
+ TimelineClusterMetric timelineClusterMetric = new TimelineClusterMetric(metric, app, null, -1l);
+ byte[] uuid = strategy.computeUuid(timelineClusterMetric, 16);
+ Assert.assertNotNull(uuid);
+ Assert.assertTrue(uuid.length == 16);
+ String uuidStr = new String(uuid);
+ Assert.assertFalse(uuids.containsKey(uuidStr) && !uuids.containsValue(timelineClusterMetric));
+ if (uuids.containsKey(uuidStr) ) {
+ if (!uuids.containsValue(timelineClusterMetric)) {
+ System.out.println("COLLISION : " + timelineClusterMetric.toString() + " = " + uuids.get(uuidStr));
+ }
+ }
+ uuids.put(uuidStr, timelineClusterMetric);
+ }
+ }
+ }
+
+ @Test
+ public void testHaseBasedUuidForAppIds() throws SQLException {
+
+ MetricUuidGenStrategy strategy = new HashBasedUuidGenStrategy();
+ Map<String, TimelineClusterMetric> uuids = new HashMap<>();
+ for (String app : metricSet.keySet()) {
+ TimelineClusterMetric timelineClusterMetric = new TimelineClusterMetric("TestMetric", app, null, -1l);
+ byte[] uuid = strategy.computeUuid(timelineClusterMetric, 16);
+ String uuidStr = new String(uuid);
+ if (uuids.containsKey(uuidStr) ) {
+ if (!uuids.containsValue(timelineClusterMetric)) {
+ System.out.println("COLLISION : " + timelineClusterMetric.toString() + " = " + uuids.get(uuidStr));
+ }
+ }
+ uuids.put(uuidStr, timelineClusterMetric);
+ }
+ }
+
+ @Test
+ public void testHashBasedUuidForHostnames() throws SQLException {
+
+ MetricUuidGenStrategy strategy = new HashBasedUuidGenStrategy();
+ Map<String, String> uuids = new HashMap<>();
+
+ List<String> hosts = new ArrayList<>();
+ String hostPrefix = "TestHost.";
+ String hostSuffix = ".ambari.apache.org";
+
+ for (int i=0; i<=2000; i++) {
+ hosts.add(hostPrefix + i + hostSuffix);
+ }
+
+ for (String host : hosts) {
+ byte[] uuid = strategy.computeUuid(host, 4);
+ Assert.assertNotNull(uuid);
+ Assert.assertTrue(uuid.length == 4);
+ String uuidStr = new String(uuid);
+ Assert.assertFalse(uuids.containsKey(uuidStr));
+ uuids.put(uuidStr, host);
+ }
+ }
+
+
+ @Test
+ public void testRandomUuidForWhitelistedMetrics() throws SQLException {
+
+ MetricUuidGenStrategy strategy = new RandomUuidGenStrategy();
+ Map<String, String> uuids = new HashMap<>();
+ for (String app : metricSet.keySet()) {
+ Set<String> metrics = metricSet.get(app);
+ for (String metric : metrics) {
+ byte[] uuid = strategy.computeUuid(new TimelineClusterMetric(metric, app, null, -1l), 16);
+ Assert.assertNotNull(uuid);
+ Assert.assertTrue(uuid.length == 16);
+ String uuidStr = new String(uuid);
+ Assert.assertFalse(uuids.containsKey(uuidStr) && !uuids.containsValue(metric));
+ uuids.put(uuidStr, metric);
+ }
+ }
+ }
+
+ public Map<String, Set<String>> populateMetricWhitelistFromFile() {
+
+
+ Map<String, Set<String>> metricSet = new HashMap<String, Set<String>>();
+ FileInputStream fstream = null;
+ BufferedReader br = null;
+ String strLine;
+ for (String appId : apps) {
+ URL fileUrl = ClassLoader.getSystemResource("metrics_def/" + appId.toUpperCase() + ".dat");
+
+ Set<String> metricsForApp = new HashSet<>();
+ try {
+ fstream = new FileInputStream(fileUrl.getPath());
+ br = new BufferedReader(new InputStreamReader(fstream));
+ while ((strLine = br.readLine()) != null) {
+ strLine = strLine.trim();
+ metricsForApp.add(strLine);
+ }
+ } catch (Exception ioEx) {
+ System.out.println("Metrics for AppId " + appId + " not found.");
+ } finally {
+ if (br != null) {
+ try {
+ br.close();
+ } catch (IOException e) {
+ }
+ }
+
+ if (fstream != null) {
+ try {
+ fstream.close();
+ } catch (IOException e) {
+ }
+ }
+ }
+ metricsForApp.add("live_hosts");
+ metricSet.put(appId.contains("hbase") ? "hbase" : appId, metricsForApp);
+ System.out.println("Found " + metricsForApp.size() + " metrics for appId = " + appId);
+ }
+ return metricSet;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/timeline/TestGenericObjectMapper.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/timeline/TestGenericObjectMapper.java b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/timeline/TestGenericObjectMapper.java
new file mode 100644
index 0000000..4a1bd5a
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/timeline/TestGenericObjectMapper.java
@@ -0,0 +1,102 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.metrics.timeline;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.io.WritableComparator;
+import org.apache.ambari.metrics.timeline.GenericObjectMapper;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import static org.junit.Assert.assertEquals;
+
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+public class TestGenericObjectMapper {
+
+ @Test
+ public void testEncoding() {
+ testEncoding(Long.MAX_VALUE);
+ testEncoding(Long.MIN_VALUE);
+ testEncoding(0l);
+ testEncoding(128l);
+ testEncoding(256l);
+ testEncoding(512l);
+ testEncoding(-256l);
+ }
+
+ private static void testEncoding(long l) {
+ byte[] b = GenericObjectMapper.writeReverseOrderedLong(l);
+ assertEquals("error decoding", l,
+ GenericObjectMapper.readReverseOrderedLong(b, 0));
+ byte[] buf = new byte[16];
+ System.arraycopy(b, 0, buf, 5, 8);
+ assertEquals("error decoding at offset", l,
+ GenericObjectMapper.readReverseOrderedLong(buf, 5));
+ if (l > Long.MIN_VALUE) {
+ byte[] a = GenericObjectMapper.writeReverseOrderedLong(l-1);
+ assertEquals("error preserving ordering", 1,
+ WritableComparator.compareBytes(a, 0, a.length, b, 0, b.length));
+ }
+ if (l < Long.MAX_VALUE) {
+ byte[] c = GenericObjectMapper.writeReverseOrderedLong(l+1);
+ assertEquals("error preserving ordering", 1,
+ WritableComparator.compareBytes(b, 0, b.length, c, 0, c.length));
+ }
+ }
+
+ private static void verify(Object o) throws IOException {
+ assertEquals(o, GenericObjectMapper.read(GenericObjectMapper.write(o)));
+ }
+
+ @Test
+ public void testValueTypes() throws IOException {
+ verify(Integer.MAX_VALUE);
+ verify(Integer.MIN_VALUE);
+ assertEquals(Integer.MAX_VALUE, GenericObjectMapper.read(
+ GenericObjectMapper.write((long) Integer.MAX_VALUE)));
+ assertEquals(Integer.MIN_VALUE, GenericObjectMapper.read(
+ GenericObjectMapper.write((long) Integer.MIN_VALUE)));
+ verify((long)Integer.MAX_VALUE + 1l);
+ verify((long)Integer.MIN_VALUE - 1l);
+
+ verify(Long.MAX_VALUE);
+ verify(Long.MIN_VALUE);
+
+ assertEquals(42, GenericObjectMapper.read(GenericObjectMapper.write(42l)));
+ verify(42);
+ verify(1.23);
+ verify("abc");
+ verify(true);
+ List<String> list = new ArrayList<String>();
+ list.add("123");
+ list.add("abc");
+ verify(list);
+ Map<String,String> map = new HashMap<String,String>();
+ map.put("k1","v1");
+ map.put("k2","v2");
+ verify(map);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/webapp/TestTimelineWebServices.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/webapp/TestTimelineWebServices.java b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/webapp/TestTimelineWebServices.java
new file mode 100644
index 0000000..3456af6
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/webapp/TestTimelineWebServices.java
@@ -0,0 +1,118 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.metrics.webapp;
+
+import static org.junit.Assert.assertEquals;
+
+import javax.ws.rs.core.MediaType;
+
+import org.apache.hadoop.metrics2.sink.timeline.TimelineMetrics;
+import org.apache.ambari.metrics.core.timeline.TestTimelineMetricStore;
+import org.apache.ambari.metrics.core.timeline.TimelineMetricStore;
+import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
+import org.apache.hadoop.yarn.webapp.YarnJacksonJaxbJsonProvider;
+import org.junit.Test;
+
+import com.google.inject.Guice;
+import com.google.inject.Injector;
+import com.google.inject.servlet.GuiceServletContextListener;
+import com.google.inject.servlet.ServletModule;
+import com.sun.jersey.api.client.ClientResponse;
+import com.sun.jersey.api.client.WebResource;
+import com.sun.jersey.api.client.config.DefaultClientConfig;
+import com.sun.jersey.guice.spi.container.servlet.GuiceContainer;
+import com.sun.jersey.test.framework.JerseyTest;
+import com.sun.jersey.test.framework.WebAppDescriptor;
+
+import junit.framework.Assert;
+
+
+public class TestTimelineWebServices extends JerseyTest {
+ private static TimelineMetricStore metricStore;
+ private long beforeTime;
+
+ private Injector injector = Guice.createInjector(new ServletModule() {
+
+ @Override
+ protected void configureServlets() {
+ bind(YarnJacksonJaxbJsonProvider.class);
+ bind(TimelineWebServices.class);
+ bind(GenericExceptionHandler.class);
+ try {
+ metricStore = new TestTimelineMetricStore();
+ } catch (Exception e) {
+ Assert.fail();
+ }
+ bind(TimelineMetricStore.class).toInstance(metricStore);
+ serve("/*").with(GuiceContainer.class);
+ }
+
+ });
+
+ public class GuiceServletConfig extends GuiceServletContextListener {
+
+ @Override
+ protected Injector getInjector() {
+ return injector;
+ }
+ }
+
+ public TestTimelineWebServices() {
+ super(new WebAppDescriptor.Builder(
+ "org.apache.ambari.metrics.webapp")
+ .contextListenerClass(GuiceServletConfig.class)
+ .filterClass(com.google.inject.servlet.GuiceFilter.class)
+ .contextPath("jersey-guice-filter")
+ .servletPath("/")
+ .clientConfig(new DefaultClientConfig(YarnJacksonJaxbJsonProvider.class))
+ .build());
+ }
+
+ @Test
+ public void testAbout() throws Exception {
+ WebResource r = resource();
+ ClientResponse response = r.path("ws").path("v1").path("timeline")
+ .accept(MediaType.APPLICATION_JSON)
+ .get(ClientResponse.class);
+ assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
+ TimelineWebServices.AboutInfo about =
+ response.getEntity(TimelineWebServices.AboutInfo.class);
+ Assert.assertNotNull(about);
+ Assert.assertEquals("AMS API", about.getAbout());
+ }
+
+ private static void verifyMetrics(TimelineMetrics metrics) {
+ Assert.assertNotNull(metrics);
+ Assert.assertEquals("cpu_user", metrics.getMetrics().get(0).getMetricName());
+ Assert.assertEquals(3, metrics.getMetrics().get(0).getMetricValues().size());
+ Assert.assertEquals("mem_free", metrics.getMetrics().get(1).getMetricName());
+ Assert.assertEquals(3, metrics.getMetrics().get(1).getMetricValues().size());
+ }
+
+ @Test
+ public void testGetMetrics() throws Exception {
+ WebResource r = resource();
+ ClientResponse response = r.path("ws").path("v1").path("timeline")
+ .path("metrics").queryParam("metricNames", "cpu_user").queryParam("precision", "seconds")
+ .accept(MediaType.APPLICATION_JSON)
+ .get(ClientResponse.class);
+ assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
+ verifyMetrics(response.getEntity(TimelineMetrics.class));
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/data/TestAppMetrics.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/data/TestAppMetrics.java b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/data/TestAppMetrics.java
deleted file mode 100644
index 499dab6..0000000
--- a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/data/TestAppMetrics.java
+++ /dev/null
@@ -1,134 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.loadsimulator.data;
-
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.loadsimulator.util.Json;
-import org.junit.Before;
-import org.junit.Test;
-
-import java.io.IOException;
-
-import static org.junit.Assert.assertEquals;
-
-public class TestAppMetrics {
- private static final String SAMPLE_SINGLE_METRIC_HOST_JSON = "{\n" +
- " \"metrics\" : [ {\n" +
- " \"instanceid\" : \"\",\n" +
- " \"hostname\" : \"localhost\",\n" +
- " \"metrics\" : {\n" +
- " \"0\" : \"5.35\",\n" +
- " \"5000\" : \"5.35\",\n" +
- " \"10000\" : \"5.35\",\n" +
- " \"15000\" : \"5.35\"\n" +
- " },\n" +
- " \"starttime\" : \"1411663170112\",\n" +
- " \"appid\" : \"HOST\",\n" +
- " \"metricname\" : \"disk_free\"\n" +
- " } ]\n" +
- "}";
-
- private static final String SAMPLE_TWO_METRIC_HOST_JSON = "{\n" +
- " \"metrics\" : [ {\n" +
- " \"instanceid\" : \"\",\n" +
- " \"hostname\" : \"localhost\",\n" +
- " \"metrics\" : {\n" +
- " \"0\" : \"5.35\",\n" +
- " \"5000\" : \"5.35\",\n" +
- " \"10000\" : \"5.35\",\n" +
- " \"15000\" : \"5.35\"\n" +
- " },\n" +
- " \"starttime\" : \"0\",\n" +
- " \"appid\" : \"HOST\",\n" +
- " \"metricname\" : \"disk_free\"\n" +
- " }, {\n" +
- " \"instanceid\" : \"\",\n" +
- " \"hostname\" : \"localhost\",\n" +
- " \"metrics\" : {\n" +
- " \"0\" : \"94.0\",\n" +
- " \"5000\" : \"94.0\",\n" +
- " \"10000\" : \"94.0\",\n" +
- " \"15000\" : \"94.0\"\n" +
- " },\n" +
- " \"starttime\" : \"0\",\n" +
- " \"appid\" : \"HOST\",\n" +
- " \"metricname\" : \"mem_cached\"\n" +
- " } ]\n" +
- "}";
-
- private long[] timestamps;
-
- @Before
- public void setUp() throws Exception {
- timestamps = new long[4];
- timestamps[0] = 0;
- timestamps[1] = timestamps[0] + 5000;
- timestamps[2] = timestamps[1] + 5000;
- timestamps[3] = timestamps[2] + 5000;
-
- }
-
- @Test
- public void testHostDiskMetricsSerialization() throws IOException {
- long timestamp = 1411663170112L;
- AppMetrics appMetrics = new AppMetrics(new ApplicationInstance("localhost", AppID.HOST, ""), timestamp);
-
- Metric diskFree = appMetrics.createMetric("disk_free");
- double value = 5.35;
-
- diskFree.putMetric(timestamps[0], Double.toString(value));
- diskFree.putMetric(timestamps[1], Double.toString(value));
- diskFree.putMetric(timestamps[2], Double.toString(value));
- diskFree.putMetric(timestamps[3], Double.toString(value));
-
- appMetrics.addMetric(diskFree);
-
- String expected = SAMPLE_SINGLE_METRIC_HOST_JSON;
- String s = new Json(true).serialize(appMetrics);
-
- assertEquals("Serialized Host Metrics", expected, s);
- }
-
-
- @Test
- public void testSingleHostManyMetricsSerialization() throws IOException {
- AppMetrics appMetrics = new AppMetrics(new ApplicationInstance("localhost", AppID.HOST, ""), timestamps[0]);
-
- Metric diskFree = appMetrics.createMetric("disk_free");
- double value = 5.35;
- diskFree.putMetric(timestamps[0], Double.toString(value));
- diskFree.putMetric(timestamps[1], Double.toString(value));
- diskFree.putMetric(timestamps[2], Double.toString(value));
- diskFree.putMetric(timestamps[3], Double.toString(value));
-
- appMetrics.addMetric(diskFree);
-
- Metric memCache = appMetrics.createMetric("mem_cached");
- double memVal = 94;
- memCache.putMetric(timestamps[0], Double.toString(memVal));
- memCache.putMetric(timestamps[1], Double.toString(memVal));
- memCache.putMetric(timestamps[2], Double.toString(memVal));
- memCache.putMetric(timestamps[3], Double.toString(memVal));
-
- appMetrics.addMetric(memCache);
-
- String expected = SAMPLE_TWO_METRIC_HOST_JSON;
- String s = new Json(true).serialize(appMetrics);
-
- assertEquals("Serialized Host Metrics", expected, s);
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/data/TestMetric.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/data/TestMetric.java b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/data/TestMetric.java
deleted file mode 100644
index a0572a2..0000000
--- a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/data/TestMetric.java
+++ /dev/null
@@ -1,80 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.loadsimulator.data;
-
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.loadsimulator.util.Json;
-import org.junit.Test;
-
-import java.io.IOException;
-
-import static org.assertj.core.api.Assertions.assertThat;
-import static org.assertj.core.api.Assertions.entry;
-import static org.junit.Assert.assertEquals;
-
-public class TestMetric {
- private static final String SAMPLE_METRIC_IN_JSON = "{\n" +
- " \"instanceid\" : \"\",\n" +
- " \"hostname\" : \"localhost\",\n" +
- " \"metrics\" : {\n" +
- " \"0\" : \"5.35\",\n" +
- " \"5000\" : \"5.35\",\n" +
- " \"10000\" : \"5.35\",\n" +
- " \"15000\" : \"5.35\"\n" +
- " },\n" +
- " \"starttime\" : \"0\",\n" +
- " \"appid\" : \"HOST\",\n" +
- " \"metricname\" : \"disk_free\"\n" +
- "}";
-
- @Test
- public void testSerializeToJson() throws IOException {
- Metric diskOnHostMetric = new Metric(new ApplicationInstance("localhost", AppID.HOST, ""), "disk_free", 0);
-
- long timestamp = 0;
- double value = 5.35;
-
- diskOnHostMetric.putMetric(timestamp, Double.toString(value));
- diskOnHostMetric.putMetric(timestamp + 5000, Double.toString(value));
- diskOnHostMetric.putMetric(timestamp + 10000, Double.toString(value));
- diskOnHostMetric.putMetric(timestamp + 15000, Double.toString(value));
-
- String expected = SAMPLE_METRIC_IN_JSON;
- String s = new Json(true).serialize(diskOnHostMetric);
-
- assertEquals("Json should match", expected, s);
- }
-
- @Test
- public void testDeserializeObjectFromString() throws IOException {
- String source = SAMPLE_METRIC_IN_JSON;
-
- Metric m = new Json().deserialize(source, Metric.class);
-
- assertEquals("localhost", m.getHostname());
- assertEquals("HOST", m.getAppid());
- assertEquals("", m.getInstanceid());
- assertEquals("disk_free", m.getMetricname());
- assertEquals("0", m.getStarttime());
-
- assertThat(m.getMetrics()).isNotEmpty().hasSize(4).contains(
- entry("0", "5.35"),
- entry("5000", "5.35"),
- entry("10000", "5.35"),
- entry("15000", "5.35"));
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/jmetertest/jmetertest/AMSJMeterLoadTest.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/jmetertest/jmetertest/AMSJMeterLoadTest.java b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/jmetertest/jmetertest/AMSJMeterLoadTest.java
deleted file mode 100644
index 7f168f2..0000000
--- a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/jmetertest/jmetertest/AMSJMeterLoadTest.java
+++ /dev/null
@@ -1,198 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p/>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p/>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.loadsimulator.jmetertest.jmetertest;
-
-import org.apache.commons.lang3.StringUtils;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.loadsimulator.MetricsLoadSimulator;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.BufferedReader;
-import java.io.FileInputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Properties;
-import java.util.concurrent.Executors;
-import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.ScheduledFuture;
-import java.util.concurrent.TimeUnit;
-
-public class AMSJMeterLoadTest {
-
- private final static Logger LOG = LoggerFactory.getLogger(AMSJMeterLoadTest.class);
- private static String PROPERTIES_FILE = "loadsimulator/ams-jmeter.properties";
- private ScheduledExecutorService scheduledExecutorService = null;
- private List<AppGetMetric> appGetMetrics;
- private Properties amsJmeterProperties = null;
-
- public AMSJMeterLoadTest(Map<String, String> args) {
-
- String testType = args.get("type");
- String userDefinedPropertiesFile = args.get("amsJmeterPropertiesFile");
- if (null == userDefinedPropertiesFile || userDefinedPropertiesFile.isEmpty()) {
- this.amsJmeterProperties = readProperties(PROPERTIES_FILE);
- } else {
- this.amsJmeterProperties = readProperties(userDefinedPropertiesFile);
- }
-
- if ("U".equals(testType)) { //GET metrics simulator
- int numInstances = Integer.valueOf(amsJmeterProperties.getProperty("num-ui-instances"));
- this.scheduledExecutorService = Executors.newScheduledThreadPool(numInstances);
- this.appGetMetrics = initializeGetMetricsPayload(amsJmeterProperties);
- this.runTest(numInstances);
- } else { //PUT Metrics simulator
- Map<String, String> mapArgs = new HashMap<String, String>();
- mapArgs.put("hostName", (args.get("host-prefix") != null) ? args.get("host-prefix") : amsJmeterProperties.getProperty("host-prefix"));
- mapArgs.put("minHostIndex", (args.get("min-host-index") != null) ? args.get("min-host-index") : amsJmeterProperties.getProperty("min-host-index"));
- mapArgs.put("numberOfHosts", (args.get("num-hosts") != null) ? args.get("num-hosts") : amsJmeterProperties.getProperty("num-hosts"));
- mapArgs.put("metricsHostName", (args.get("ams-host-port") != null) ? args.get("ams-host-port") : amsJmeterProperties.getProperty("ams-host-port"));
- mapArgs.put("collectInterval", (args.get("collection-interval") != null) ? args.get("collection-interval") : amsJmeterProperties.getProperty("collection-interval"));
- mapArgs.put("sendInterval", (args.get("send-interval") != null) ? args.get("send-interval") : amsJmeterProperties.getProperty("send-interval"));
- mapArgs.put("master", (args.get("create-master") != null) ? args.get("create-master") : amsJmeterProperties.getProperty("create-master"));
- System.out.println("AMS Load Simulation Parameters : " + mapArgs);
- MetricsLoadSimulator.startTest(mapArgs);
- }
- }
-
- public static Properties readProperties(String propertiesFile) {
- try {
- Properties properties = new Properties();
- InputStream inputStream = ClassLoader.getSystemResourceAsStream(propertiesFile);
- if (inputStream == null) {
- inputStream = new FileInputStream(propertiesFile);
- }
- properties.load(inputStream);
- return properties;
- } catch (IOException ioEx) {
- LOG.error("Error reading properties file for jmeter");
- return null;
- }
- }
-
- private static List<GetMetricRequestInfo> readMetricsFromFile(String app) {
- InputStream input = null;
- List<GetMetricRequestInfo> metricList = new ArrayList<>();
- String fileName = "ui_metrics_def/" + app + ".dat";
-
- try {
- input = ClassLoader.getSystemResourceAsStream(fileName);
- BufferedReader reader = new BufferedReader(new InputStreamReader(input));
- String line;
- List<String> metrics = new ArrayList<>();
- while ((line = reader.readLine()) != null) {
-
- if (line.startsWith("|")) {
- boolean needsTimestamps = line.contains("startTime");
- boolean needsHost = line.contains("hostname");
- metricList.add(new GetMetricRequestInfo(metrics, needsTimestamps, needsHost));
- metrics.clear();
- } else {
- metrics.add(line);
- }
- }
- return metricList;
- } catch (IOException e) {
- LOG.error("Cannot read file " + fileName + " for appID " + app, e);
- } finally {
- if (input != null) {
- try {
- input.close();
- } catch (IOException ex) {
- }
- }
- }
- return null;
- }
-
- private static List<AppGetMetric> initializeGetMetricsPayload(Properties amsJmeterProperties) {
-
- List<AppGetMetric> appGetMetrics = new ArrayList<AppGetMetric>();
- String appsToTest = amsJmeterProperties.getProperty("apps-to-test");
- String[] apps;
-
- if (appsToTest != null && !appsToTest.isEmpty()) {
- apps = StringUtils.split(appsToTest, ",");
- } else {
- apps = new String[JmeterTestPlanTask.ClientApp.values().length];
- int ctr = 0;
- for (JmeterTestPlanTask.ClientApp app : JmeterTestPlanTask.ClientApp.values())
- apps[ctr++] = app.getId();
- }
-
- for (String app : apps) {
-
- int interval = Integer.valueOf(amsJmeterProperties.getProperty("get-interval"));
- String intervalString = amsJmeterProperties.getProperty(app + "-get-interval");
- if (intervalString != null && !intervalString.isEmpty()) {
- interval = Integer.valueOf(intervalString);
- }
- appGetMetrics.add(new AppGetMetric(readMetricsFromFile(app), interval, app));
- }
-
- return appGetMetrics;
- }
-
- public void runTest(int numInstances) {
-
- int appRefreshRate = Integer.valueOf(amsJmeterProperties.getProperty("app-refresh-rate"));
- for (int i = 0; i < numInstances; i++) {
- ScheduledFuture future = scheduledExecutorService.scheduleAtFixedRate(new JmeterTestPlanTask(appGetMetrics,
- amsJmeterProperties), 0, appRefreshRate, TimeUnit.MILLISECONDS);
- }
- }
-
- /**
- * Sample Usage:
- * java -cp "lib/*":ambari-metrics-timelineservice-2.1.1.0.jar org.apache.hadoop.yarn.server.applicationhistoryservice
- * .metrics.loadsimulator.jmeter.AMSJMeterLoadTest
- * -t UI -p ambari-metrics-timelineservice/src/main/resources/jmeter/ams-jmeter.properties
- */
- public static void main(String[] args) {
- Map<String, String> mapArgs = parseArgs(args);
- new AMSJMeterLoadTest(mapArgs);
- }
-
- private static Map<String, String> parseArgs(String[] args) {
- Map<String, String> mapProps = new HashMap<String, String>();
- if (args.length == 0) {
- printUsage();
- throw new RuntimeException("Unexpected argument, See usage message.");
- } else {
- for (int i = 0; i < args.length; i += 2) {
- String arg = args[i];
- mapProps.put(arg.substring(1), args[i+1]);
- }
- }
- return mapProps;
- }
-
- public static void printUsage() {
- System.err.println("Usage: java AMSJmeterLoadTest [OPTIONS]");
- System.err.println("Options: ");
- System.err.println("[--t type (S=>Sink/U=>UI)] [-ams-host-port localhost:6188] [-min-host-index 2] [-host-prefix TestHost.] [-num-hosts 2] " +
- "[-create-master true] [-collection-interval 10000 ] [-send-interval 60000 ] [-p amsJmeterPropertiesFile (Optional)]");
- }
-
-}
-
-
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/jmetertest/jmetertest/AppGetMetric.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/jmetertest/jmetertest/AppGetMetric.java b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/jmetertest/jmetertest/AppGetMetric.java
deleted file mode 100644
index a12cb4a..0000000
--- a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/jmetertest/jmetertest/AppGetMetric.java
+++ /dev/null
@@ -1,57 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p/>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p/>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.loadsimulator.jmetertest.jmetertest;
-
-import java.util.List;
-
-public class AppGetMetric {
-
- private String app;
- private int interval;
- private List<GetMetricRequestInfo> requests;
-
- public AppGetMetric(List<GetMetricRequestInfo> requests, int interval, String app) {
- this.setMetricRequests(requests);
- this.setInterval(interval);
- this.setApp(app);
- }
-
- public List<GetMetricRequestInfo> getMetricRequests() {
- return requests;
- }
-
- public void setMetricRequests(List<GetMetricRequestInfo> requests) {
- this.requests = requests;
- }
-
- public int getInterval() {
- return interval;
- }
-
- public void setInterval(int interval) {
- this.interval = interval;
- }
-
- public String getApp() {
- return app;
- }
-
- public void setApp(String app) {
- this.app = app;
- }
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/jmetertest/jmetertest/GetMetricRequestInfo.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/jmetertest/jmetertest/GetMetricRequestInfo.java b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/jmetertest/jmetertest/GetMetricRequestInfo.java
deleted file mode 100644
index 4bd44d9..0000000
--- a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/jmetertest/jmetertest/GetMetricRequestInfo.java
+++ /dev/null
@@ -1,61 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.loadsimulator.jmetertest.jmetertest;
-
-import org.apache.commons.lang.StringUtils;
-
-import java.util.List;
-
-
-public class GetMetricRequestInfo {
-
- private String metricStringPayload;
- private boolean needsTimestamps;
- private boolean needsHost;
-
- public GetMetricRequestInfo(List<String> metrics, boolean needsTimestamps, boolean needsHost) {
-
- this.setMetricStringPayload(StringUtils.join(metrics, ","));
- this.setNeedsTimestamps(needsTimestamps);
- this.setNeedsHost(needsHost);
- }
-
- public String getMetricStringPayload() {
- return metricStringPayload;
- }
-
- public void setMetricStringPayload(String metricStringPayload) {
- this.metricStringPayload = metricStringPayload;
- }
-
- public boolean needsTimestamps() {
- return needsTimestamps;
- }
-
- public void setNeedsTimestamps(boolean needsTimestamps) {
- this.needsTimestamps = needsTimestamps;
- }
-
- public boolean needsHost() {
- return needsHost;
- }
-
- public void setNeedsHost(boolean needsHost) {
- this.needsHost = needsHost;
- }
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/jmetertest/jmetertest/JmeterTestPlanTask.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/jmetertest/jmetertest/JmeterTestPlanTask.java b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/jmetertest/jmetertest/JmeterTestPlanTask.java
deleted file mode 100644
index c6df162..0000000
--- a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/jmetertest/jmetertest/JmeterTestPlanTask.java
+++ /dev/null
@@ -1,276 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p/>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p/>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.loadsimulator.jmetertest.jmetertest;
-
-import org.apache.commons.io.IOUtils;
-import org.apache.jmeter.control.LoopController;
-import org.apache.jmeter.engine.StandardJMeterEngine;
-import org.apache.jmeter.protocol.http.sampler.HTTPSampler;
-import org.apache.jmeter.protocol.http.util.HTTPConstants;
-import org.apache.jmeter.reporters.ResultCollector;
-import org.apache.jmeter.reporters.Summariser;
-import org.apache.jmeter.testelement.TestElement;
-import org.apache.jmeter.testelement.TestPlan;
-import org.apache.jmeter.threads.JMeterContextService;
-import org.apache.jmeter.threads.ThreadGroup;
-import org.apache.jmeter.timers.ConstantTimer;
-import org.apache.jmeter.util.JMeterUtils;
-import org.apache.jorphan.collections.HashTree;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.FileInputStream;
-import java.io.FileOutputStream;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.lang.reflect.Field;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Properties;
-import java.util.Random;
-
-public class JmeterTestPlanTask implements Runnable {
-
- private static StandardJMeterEngine jmeterEngine = null;
- private final static Logger LOG = LoggerFactory.getLogger(JmeterTestPlanTask.class);
- private List<AppGetMetric> appGetMetrics;
- private Properties amsJmeterProperties;
- private HashTree amsTestPlanTree;
- private TestPlan amsTestPlan;
- private static final String JMETER_HOME = "loadsimulator";
- private static final String JMETER_PROPERTIES_FILE = JMETER_HOME + "/jmeter.properties";
- private static final String SAVESERVICE_PROPERTIES_FILE = JMETER_HOME + "/saveservice.properties";
-
- public enum ClientApp {
- HOST("HOST"),
- NAMENODE("NAMENODE"),
- HBASE("HBASE"),
- NIMBUS("NIMBUS"),
- KAFKA_BROKER("KAFKA_BROKER"),
- FLUME_HANDLER("FLUME_HANDLER"),
- AMS_HBASE("AMS-HBASE"),
- NODEMANAGER("NODEMANAGER"),
- RESOURCEMANAGER("RESOURCEMANAGER"),
- DATANODE("DATANODE");
-
- private String id;
-
- private ClientApp(String id) {
- this.id = id;
- }
-
- public String getId() {
- return id;
- }
- }
-
- public JmeterTestPlanTask(List<AppGetMetric> appGetMetrics, Properties amsJmeterProperties) {
- this.appGetMetrics = appGetMetrics;
- this.amsJmeterProperties = amsJmeterProperties;
- amsTestPlanTree = new HashTree();
- amsTestPlan = new TestPlan("AMS JMeter Load Test plan");
- System.out.println("Starting AMS Jmeter load testing");
- }
-
- public void run() {
- if (jmeterEngine != null) {
-
- Object[] threadGroups = amsTestPlanTree.getArray(amsTestPlan);
- for (Object threadGroupObj : threadGroups) {
- if (threadGroupObj instanceof ThreadGroup) {
- ThreadGroup threadGroup = (ThreadGroup) threadGroupObj;
- threadGroup.stop();
- }
- }
- amsTestPlanTree.clear();
- jmeterEngine.askThreadsToStop();
- jmeterEngine.stopTest();
- JMeterContextService.endTest();
- }
-
- //Start the new test plan for the new app.
- try {
- //Initialize Jmeter essentials
- jmeterEngine = new StandardJMeterEngine();
- JMeterContextService.getContext().setEngine(jmeterEngine);
-
- //Workaround to supply JMeterUtils with jmeter.prooperties from JAR.
- JMeterUtils.setJMeterHome("");
- Field f = new JMeterUtils().getClass().getDeclaredField("appProperties");
- f.setAccessible(true);
- f.set(null, AMSJMeterLoadTest.readProperties(JMETER_PROPERTIES_FILE));
-
- //Copy saveservices.properties file to tmp dir for JMeter to consume.
- InputStream inputStream = ClassLoader.getSystemResourceAsStream(SAVESERVICE_PROPERTIES_FILE);
- if (inputStream == null) {
- inputStream = new FileInputStream(SAVESERVICE_PROPERTIES_FILE);
- }
- String tmpDir = System.getProperty("java.io.tmpdir");
- OutputStream outputStream = new FileOutputStream(tmpDir + "/saveservice.properties");
- IOUtils.copy(inputStream, outputStream);
- outputStream.close();
- JMeterUtils.setProperty("saveservice_properties", tmpDir + "/saveservice.properties");
-
- //Initialize Test plan
- amsTestPlan.setProperty(TestElement.TEST_CLASS, TestPlan.class.getName());
- amsTestPlanTree.add("AMS Test plan", amsTestPlan);
-
- //Choose a random APP to run the perform GET metrics request.
- int currentAppIndex = new Random().nextInt(appGetMetrics.size());
-
- //Create ThreadGroup for the App
- createThreadGroupHashTree(currentAppIndex, amsJmeterProperties, amsTestPlanTree, amsTestPlan);
-
- //Geneates the JMX file that you can use through the GUI mode.
- //SaveService.saveTree(amsTestPlanTree, new FileOutputStream(JMETER_HOME + "/" + "amsTestPlan.jmx"));
-
- //Summarizer output to get test progress in stdout like.
- Summariser summariser = null;
- String summariserName = JMeterUtils.getPropDefault("summariser.name", "summary");
- if (summariserName.length() > 0) {
- summariser = new Summariser(summariserName);
- }
-
- //Store execution results into a .jtl file
- String jmeterLogFile = tmpDir + "/amsJmeterTestResults.jtl";
- ResultCollector resultCollector = new ResultCollector(summariser);
- resultCollector.setFilename(jmeterLogFile);
- amsTestPlanTree.add(amsTestPlanTree.getArray()[0], resultCollector);
- jmeterEngine.configure(amsTestPlanTree);
- jmeterEngine.run();
-
- LOG.info("AMS Jmeter Test started up successfully");
-
- } catch (Exception ioEx) {
- amsTestPlanTree.clear();
- jmeterEngine.askThreadsToStop();
- jmeterEngine.stopTest();
- JMeterContextService.endTest();
- LOG.error("Error occurred while running AMS load test : " + ioEx.getMessage());
- ioEx.printStackTrace();
- }
- }
-
- private ConstantTimer createConstantTimer(int delay) {
- ConstantTimer timer = new ConstantTimer();
- timer.setDelay("" + delay);
- return timer;
- }
-
- private Map<String, String> getAppSpecificParameters(String app, GetMetricRequestInfo request, Properties amsJmeterProperties) {
-
- Map<String, String> parametersMap = new HashMap<String, String>();
- String hostPrefix = amsJmeterProperties.getProperty("host-prefix");
- String hostSuffix = amsJmeterProperties.getProperty("host-suffix");
- int minHostIndex = Integer.valueOf(amsJmeterProperties.getProperty("min-host-index"));
- int numHosts = Integer.valueOf(amsJmeterProperties.getProperty("num-hosts"));
-
- parametersMap.put("appId", app);
-
- if (request.needsTimestamps()) {
- long currentTime = System.currentTimeMillis();
- long oneHourBack = currentTime - 3600 * 1000;
- parametersMap.put("startTime", String.valueOf(oneHourBack));
- parametersMap.put("endTime", String.valueOf(currentTime));
- }
-
- if (request.needsHost()) {
- if (ClientApp.AMS_HBASE.getId().equals(app)) {
- parametersMap.put("hostname", amsJmeterProperties.getProperty("ams-host"));
- } else if (ClientApp.HOST.getId().equals(app) || ClientApp.NODEMANAGER.getId().equals(app)) {
- int randomHost = minHostIndex + new Random().nextInt(numHosts);
- parametersMap.put("hostname", hostPrefix + randomHost + hostSuffix);
- } else {
- parametersMap.put("hostname", hostPrefix + amsJmeterProperties.getProperty(app + "-host") + hostSuffix);
- }
- }
- parametersMap.put("metricNames", request.getMetricStringPayload());
- return parametersMap;
- }
-
- private void createThreadGroupHashTree(int appIndex, Properties amsJmeterProperties, HashTree amsTestPlanTree, TestPlan amsTestPlan) {
-
- AppGetMetric appGetMetric = appGetMetrics.get(appIndex);
- String app = appGetMetric.getApp();
- int interval = appGetMetric.getInterval();
-
- //Read and validate AMS information.
- String[] amsHostPort = amsJmeterProperties.getProperty("ams-host-port").split(":");
- String amsHost = amsHostPort[0];
- String amsPath = amsJmeterProperties.getProperty("ams-path");
- int amsPort = Integer.valueOf(amsHostPort[1]);
- int numLoops = Integer.valueOf(amsJmeterProperties.getProperty("num-get-calls-per-app"));
-
- LoopController loopController = createLoopController(app + " GET loop controller", numLoops, false);
- for (GetMetricRequestInfo request : appGetMetric.getMetricRequests()) {
-
- ThreadGroup threadGroup = createThreadGroup(app + " GET threadGroup", 1, 0, loopController);
-
- HashTree threadGroupHashTree = amsTestPlanTree.add(amsTestPlan, threadGroup);
- Map<String, String> parametersMap = getAppSpecificParameters(app, request, amsJmeterProperties);
-
- HTTPSampler sampler = createGetSampler("GET " + app + " metrics", amsHost, amsPort, amsPath, null, parametersMap);
-
- if (numLoops > 1) {
- threadGroupHashTree.add(createConstantTimer(interval));
- }
-
- threadGroupHashTree.add(sampler);
- }
- }
-
- private HTTPSampler createGetSampler(String name, String domain, int port, String path, String encoding, Map<String, String> parameters) {
-
- HTTPSampler sampler = new HTTPSampler();
- sampler.setDomain(domain);
- sampler.setPort(port);
- sampler.setPath(path);
- sampler.setMethod(HTTPConstants.GET);
-
- if (encoding != null)
- sampler.setContentEncoding(encoding);
-
- for (Map.Entry<String, String> entry : parameters.entrySet()) {
- sampler.addArgument(entry.getKey(), entry.getValue());
- }
- sampler.setName(name);
- return sampler;
- }
-
- private LoopController createLoopController(String name, int numLoops, boolean continueForever) {
- LoopController loopController = new LoopController();
- loopController.setLoops(numLoops);
- loopController.setProperty(TestElement.TEST_CLASS, LoopController.class.getName());
- loopController.initialize();
- loopController.setContinueForever(continueForever);
- loopController.setName(name);
- return loopController;
- }
-
- private ThreadGroup createThreadGroup(String name, int numThreads, int rampUp, LoopController loopController) {
- ThreadGroup threadGroup = new ThreadGroup();
- threadGroup.setName(name);
- threadGroup.setNumThreads(numThreads);
- threadGroup.setRampUp(rampUp);
- threadGroup.setSamplerController(loopController);
- threadGroup.setProperty(TestElement.TEST_CLASS, ThreadGroup.class.getName());
- return threadGroup;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/net/TestRestMetricsSender.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/net/TestRestMetricsSender.java b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/net/TestRestMetricsSender.java
deleted file mode 100644
index 4411be5..0000000
--- a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/net/TestRestMetricsSender.java
+++ /dev/null
@@ -1,76 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics
- .loadsimulator.net;
-
-import org.junit.Test;
-
-import java.io.IOException;
-
-import static org.easymock.EasyMock.*;
-import static org.junit.Assert.assertEquals;
-
-public class TestRestMetricsSender {
-
- @Test
- public void testPushMetrics() throws Exception {
- final UrlService svcMock = createStrictMock(UrlService.class);
- final String payload = "test";
- final String expectedResponse = "mockResponse";
-
- expect(svcMock.send(anyString())).andReturn(expectedResponse);
- svcMock.disconnect();
- expectLastCall();
-
- replay(svcMock);
-
- RestMetricsSender sender = new RestMetricsSender("expectedHostName") {
- @Override
- protected UrlService getConnectedUrlService() throws IOException {
- return svcMock;
- }
- };
- String response = sender.pushMetrics(payload);
-
- verify(svcMock);
- assertEquals("", expectedResponse, response);
- }
-
- @Test
- public void testPushMetricsFailed() throws Exception {
- final UrlService svcMock = createStrictMock(UrlService.class);
- final String payload = "test";
- final String expectedResponse = "mockResponse";
- RestMetricsSender sender = new RestMetricsSender("expectedHostName") {
- @Override
- protected UrlService getConnectedUrlService() throws IOException {
- return svcMock;
- }
- };
-
- expect(svcMock.send(anyString())).andThrow(new IOException());
- svcMock.disconnect();
- expectLastCall();
-
- replay(svcMock);
-
- String response = sender.pushMetrics(payload);
-
- verify(svcMock);
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/net/TestStdOutMetricsSender.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/net/TestStdOutMetricsSender.java b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/net/TestStdOutMetricsSender.java
deleted file mode 100644
index 7e29ae3..0000000
--- a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/net/TestStdOutMetricsSender.java
+++ /dev/null
@@ -1,37 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.loadsimulator.net;
-
-
-import org.junit.Test;
-
-import java.io.ByteArrayOutputStream;
-import java.io.PrintStream;
-
-public class TestStdOutMetricsSender {
-
- @Test
- public void testPushMetrics() throws Exception {
- ByteArrayOutputStream baos = new ByteArrayOutputStream();
- PrintStream out = new PrintStream(baos);
- StdOutMetricsSender sender = new StdOutMetricsSender("expectedHostName", out);
- sender.pushMetrics("test");
-
- System.out.println(baos.toString());
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/util/TestRandomMetricsProvider.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/util/TestRandomMetricsProvider.java b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/util/TestRandomMetricsProvider.java
deleted file mode 100644
index 462aaf0..0000000
--- a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/util/TestRandomMetricsProvider.java
+++ /dev/null
@@ -1,36 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.loadsimulator.util;
-
-import org.junit.Test;
-
-import static org.junit.Assert.assertTrue;
-
-public class TestRandomMetricsProvider {
-
- @Test
- public void testReturnSingle() {
- double from = 5.25;
- double to = 5.40;
- RandomMetricsProvider provider = new RandomMetricsProvider(from, to);
- double metric = provider.next();
-
- assertTrue("Generated metric should be in range", from < metric && metric < to);
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/util/TestTimeStampProvider.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/util/TestTimeStampProvider.java b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/util/TestTimeStampProvider.java
deleted file mode 100644
index dd513aa..0000000
--- a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/util/TestTimeStampProvider.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.loadsimulator.util;
-
-import org.junit.Test;
-
-import static org.assertj.core.api.Assertions.assertThat;
-import static org.junit.Assert.assertEquals;
-
-public class TestTimeStampProvider {
-
- @Test
- public void testReturnSingle() {
- long startTime = 1411663170112L;
- int timeStep = 5000;
- TimeStampProvider tm = new TimeStampProvider(startTime, timeStep, 0);
-
- long tStamp = tm.next();
-
- assertEquals("First generated timestamp should match starttime", startTime, tStamp);
- }
-
- @Test
- public void testReturnTstampsForSendInterval() throws Exception {
- long startTime = 0;
- int collectInterval = 5;
- int sendInterval = 30;
- TimeStampProvider tsp = new TimeStampProvider(startTime, collectInterval, sendInterval);
-
- long[] timestamps = tsp.timestampsForNextInterval();
-
- assertThat(timestamps)
- .hasSize(6)
- .containsOnly(0, 5, 10, 15, 20, 25);
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/AbstractMiniHBaseClusterTest.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/AbstractMiniHBaseClusterTest.java b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/AbstractMiniHBaseClusterTest.java
deleted file mode 100644
index c4cebd6..0000000
--- a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/AbstractMiniHBaseClusterTest.java
+++ /dev/null
@@ -1,295 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline;
-
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.OUT_OFF_BAND_DATA_TIME_ALLOWANCE;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query.PhoenixTransactSQL.METRICS_RECORD_TABLE_NAME;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query.PhoenixTransactSQL.UPSERT_METRICS_SQL;
-import static org.apache.phoenix.end2end.ParallelStatsDisabledIT.tearDownMiniCluster;
-import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES;
-import static org.assertj.core.api.Assertions.assertThat;
-
-import java.io.IOException;
-import java.sql.Connection;
-import java.sql.DriverManager;
-import java.sql.PreparedStatement;
-import java.sql.ResultSet;
-import java.sql.SQLException;
-import java.sql.Statement;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Properties;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.IntegrationTestingUtility;
-import org.apache.hadoop.hbase.client.Admin;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
-import org.apache.hadoop.metrics2.sink.timeline.TimelineMetric;
-import org.apache.hadoop.metrics2.sink.timeline.TimelineMetrics;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.aggregators.AggregatorUtils;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query.PhoenixConnectionProvider;
-import org.apache.hadoop.yarn.util.timeline.TimelineUtils;
-import org.apache.log4j.Level;
-import org.apache.log4j.Logger;
-import org.apache.phoenix.hbase.index.write.IndexWriterUtils;
-import org.apache.phoenix.query.BaseTest;
-import org.apache.phoenix.query.QueryServices;
-import org.apache.phoenix.util.PropertiesUtil;
-import org.apache.phoenix.util.ReadOnlyProps;
-import org.junit.After;
-import org.junit.AfterClass;
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-public abstract class AbstractMiniHBaseClusterTest extends BaseTest {
-
- protected static final long BATCH_SIZE = 3;
- protected Connection conn;
- protected PhoenixHBaseAccessor hdb;
-
- public final Log LOG;
-
- public AbstractMiniHBaseClusterTest() {
- LOG = LogFactory.getLog(this.getClass());
- }
-
- @BeforeClass
- public static void doSetup() throws Exception {
- Map<String, String> props = getDefaultProps();
- props.put(IntegrationTestingUtility.IS_DISTRIBUTED_CLUSTER, "false");
- props.put(QueryServices.QUEUE_SIZE_ATTRIB, Integer.toString(5000));
- props.put(IndexWriterUtils.HTABLE_THREAD_KEY, Integer.toString(100));
- // Make a small batch size to test multiple calls to reserve sequences
- props.put(QueryServices.SEQUENCE_CACHE_SIZE_ATTRIB, Long.toString(BATCH_SIZE));
- // Must update config before starting server
- setUpTestDriver(new ReadOnlyProps(props.entrySet().iterator()));
- }
-
- @AfterClass
- public static void doTeardown() throws Exception {
- dropNonSystemTables();
- tearDownMiniCluster();
- }
-
- @Before
- public void setUp() throws Exception {
- Logger.getLogger("org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline").setLevel(Level.DEBUG);
- hdb = createTestableHBaseAccessor();
- // inits connection, starts mini cluster
- conn = getConnection(getUrl());
-
- hdb.initMetricSchema();
- }
-
- private void deleteTableIgnoringExceptions(Statement stmt, String tableName) {
- try {
- stmt.execute("delete from " + tableName);
- } catch (Exception e) {
- LOG.warn("Exception on delete table " + tableName, e);
- }
- }
-
- @After
- public void tearDown() {
- Connection conn = null;
- Statement stmt = null;
- try {
- conn = getConnection(getUrl());
- stmt = conn.createStatement();
-
- deleteTableIgnoringExceptions(stmt, "METRIC_AGGREGATE");
- deleteTableIgnoringExceptions(stmt, "METRIC_AGGREGATE_MINUTE");
- deleteTableIgnoringExceptions(stmt, "METRIC_AGGREGATE_HOURLY");
- deleteTableIgnoringExceptions(stmt, "METRIC_AGGREGATE_DAILY");
- deleteTableIgnoringExceptions(stmt, "METRIC_RECORD");
- deleteTableIgnoringExceptions(stmt, "METRIC_RECORD_MINUTE");
- deleteTableIgnoringExceptions(stmt, "METRIC_RECORD_HOURLY");
- deleteTableIgnoringExceptions(stmt, "METRIC_RECORD_DAILY");
- deleteTableIgnoringExceptions(stmt, "METRICS_METADATA");
- deleteTableIgnoringExceptions(stmt, "HOSTED_APPS_METADATA");
-
- conn.commit();
- } catch (Exception e) {
- LOG.warn("Error on deleting HBase schema.", e);
- } finally {
- if (stmt != null) {
- try {
- stmt.close();
- } catch (SQLException e) {
- // Ignore
- }
- }
-
- if (conn != null) {
- try {
- conn.close();
- } catch (SQLException e) {
- // Ignore
- }
- }
- }
- }
-
- public static Map<String, String> getDefaultProps() {
- Map<String, String> props = new HashMap<String, String>();
- // Must update config before starting server
- props.put(QueryServices.STATS_USE_CURRENT_TIME_ATTRIB,
- Boolean.FALSE.toString());
- props.put("java.security.krb5.realm", "");
- props.put("java.security.krb5.kdc", "");
- return props;
- }
-
- protected Connection getConnection(String url) throws SQLException {
- Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
- Connection conn = DriverManager.getConnection(getUrl(), props);
- return conn;
- }
-
- /**
- * A canary test. Will show if the infrastructure is set-up correctly.
- */
- @Test
- public void testClusterOK() throws Exception {
- Connection conn = getConnection(getUrl());
- conn.setAutoCommit(true);
-
- String sampleDDL = "CREATE TABLE TEST_METRICS " +
- "(TEST_COLUMN VARCHAR " +
- "CONSTRAINT pk PRIMARY KEY (TEST_COLUMN)) " +
- "DATA_BLOCK_ENCODING='FAST_DIFF', IMMUTABLE_ROWS=true, " +
- "TTL=86400, COMPRESSION='NONE' ";
-
- Statement stmt = conn.createStatement();
- stmt.executeUpdate(sampleDDL);
- conn.commit();
-
- ResultSet rs = stmt.executeQuery(
- "SELECT COUNT(TEST_COLUMN) FROM TEST_METRICS");
-
- rs.next();
- long l = rs.getLong(1);
- assertThat(l).isGreaterThanOrEqualTo(0);
-
- stmt.execute("DROP TABLE TEST_METRICS");
- conn.close();
- }
-
- protected PhoenixHBaseAccessor createTestableHBaseAccessor() {
- Configuration metricsConf = new Configuration();
- metricsConf.set(TimelineMetricConfiguration.HBASE_COMPRESSION_SCHEME, "NONE");
- // Unit tests insert values into the future
- metricsConf.setLong(OUT_OFF_BAND_DATA_TIME_ALLOWANCE, 600000);
-
- return
- new PhoenixHBaseAccessor(new TimelineMetricConfiguration(new Configuration(), metricsConf),
- new PhoenixConnectionProvider() {
- @Override
- public Admin getHBaseAdmin() throws IOException {
- try {
- return driver.getConnectionQueryServices(null, null).getAdmin();
- } catch (SQLException e) {
- LOG.error(e);
- }
- return null;
- }
-
- @Override
- public Connection getConnection() {
- Connection connection = null;
- try {
- connection = DriverManager.getConnection(getUrl());
- } catch (SQLException e) {
- LOG.warn("Unable to connect to HBase store using Phoenix.", e);
- }
- return connection;
- }
-
- });
- }
-
- protected void insertMetricRecords(Connection conn, TimelineMetrics metrics, long currentTime)
- throws SQLException, IOException {
-
- List<TimelineMetric> timelineMetrics = metrics.getMetrics();
- if (timelineMetrics == null || timelineMetrics.isEmpty()) {
- LOG.debug("Empty metrics insert request.");
- return;
- }
-
- PreparedStatement metricRecordStmt = null;
-
- try {
- metricRecordStmt = conn.prepareStatement(String.format(
- UPSERT_METRICS_SQL, METRICS_RECORD_TABLE_NAME));
-
- for (TimelineMetric metric : timelineMetrics) {
- metricRecordStmt.clearParameters();
-
- if (LOG.isTraceEnabled()) {
- LOG.trace("host: " + metric.getHostName() + ", " +
- "metricName = " + metric.getMetricName() + ", " +
- "values: " + metric.getMetricValues());
- }
- double[] aggregates = AggregatorUtils.calculateAggregates(
- metric.getMetricValues());
-
- metricRecordStmt.setString(1, metric.getMetricName());
- metricRecordStmt.setString(2, metric.getHostName());
- metricRecordStmt.setString(3, metric.getAppId());
- metricRecordStmt.setString(4, metric.getInstanceId());
- metricRecordStmt.setLong(5, metric.getStartTime());
- metricRecordStmt.setString(6, metric.getType());
- metricRecordStmt.setDouble(7, aggregates[0]);
- metricRecordStmt.setDouble(8, aggregates[1]);
- metricRecordStmt.setDouble(9, aggregates[2]);
- metricRecordStmt.setLong(10, (long) aggregates[3]);
- String json = TimelineUtils.dumpTimelineRecordtoJSON(metric.getMetricValues());
- metricRecordStmt.setString(11, json);
-
- try {
- metricRecordStmt.executeUpdate();
- } catch (SQLException sql) {
- LOG.error(sql);
- }
- }
-
- conn.commit();
-
- } finally {
- if (metricRecordStmt != null) {
- try {
- metricRecordStmt.close();
- } catch (SQLException e) {
- // Ignore
- }
- }
- if (conn != null) {
- try {
- conn.close();
- } catch (SQLException sql) {
- // Ignore
- }
- }
- }
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/AbstractPhoenixConnectionlessTest.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/AbstractPhoenixConnectionlessTest.java b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/AbstractPhoenixConnectionlessTest.java
deleted file mode 100644
index 1430478..0000000
--- a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/AbstractPhoenixConnectionlessTest.java
+++ /dev/null
@@ -1,111 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline;
-
-import org.apache.phoenix.jdbc.PhoenixEmbeddedDriver;
-import org.apache.phoenix.jdbc.PhoenixTestDriver;
-import org.apache.phoenix.query.BaseTest;
-import org.apache.phoenix.util.PropertiesUtil;
-import org.apache.phoenix.util.ReadOnlyProps;
-import org.apache.phoenix.util.TestUtil;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-import java.sql.Connection;
-import java.sql.DriverManager;
-import java.sql.PreparedStatement;
-import static org.apache.phoenix.util.PhoenixRuntime.TENANT_ID_ATTRIB;
-import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES;
-import static org.junit.Assert.assertNull;
-import static org.junit.Assert.assertTrue;
-
-public abstract class AbstractPhoenixConnectionlessTest extends BaseTest {
-
- protected static String getUrl() {
- return TestUtil.PHOENIX_CONNECTIONLESS_JDBC_URL;
- }
-
- protected static String getUrl(String tenantId) {
- return getUrl() + ';' + TENANT_ID_ATTRIB + '=' + tenantId;
- }
-
- protected static PhoenixTestDriver driver;
-
- private static void startServer(String url) throws Exception {
- assertNull(driver);
- // only load the test driver if we are testing locally - for integration tests, we want to
- // test on a wider scale
- if (PhoenixEmbeddedDriver.isTestUrl(url)) {
- driver = initDriver(ReadOnlyProps.EMPTY_PROPS);
- assertTrue(DriverManager.getDriver(url) == driver);
- driver.connect(url, PropertiesUtil.deepCopy(TEST_PROPERTIES));
- }
- }
-
- protected static synchronized PhoenixTestDriver initDriver(ReadOnlyProps props) throws Exception {
- if (driver == null) {
- driver = new PhoenixTestDriver(props);
- DriverManager.registerDriver(driver);
- }
- return driver;
- }
-
- private String connUrl;
-
- @Before
- public void setup() throws Exception {
- connUrl = getUrl();
- startServer(connUrl);
- }
-
- @Test
- public void testStorageSystemInitialized() throws Exception {
- String sampleDDL = "CREATE TABLE TEST_METRICS (TEST_COLUMN VARCHAR " +
- "CONSTRAINT pk PRIMARY KEY (TEST_COLUMN)) DATA_BLOCK_ENCODING='FAST_DIFF', " +
- "IMMUTABLE_ROWS=true, TTL=86400, COMPRESSION='SNAPPY'";
-
- Connection conn = null;
- PreparedStatement stmt = null;
- try {
- conn = DriverManager.getConnection(connUrl);
- stmt = conn.prepareStatement(sampleDDL);
- stmt.execute();
- conn.commit();
- } finally {
- if (stmt != null) {
- stmt.close();
- }
- if (conn != null) {
- conn.close();
- }
- }
- }
-
- @After
- public void tearDown() throws Exception {
- if (driver != null) {
- try {
- driver.close();
- } finally {
- PhoenixTestDriver phoenixTestDriver = driver;
- driver = null;
- DriverManager.deregisterDriver(phoenixTestDriver);
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/FunctionTest.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/FunctionTest.java b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/FunctionTest.java
deleted file mode 100644
index 188f634..0000000
--- a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/FunctionTest.java
+++ /dev/null
@@ -1,62 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline;
-
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.aggregators.Function;
-import org.junit.Ignore;
-import org.junit.Test;
-
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.aggregators.Function.fromMetricName;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.aggregators.Function.ReadFunction.AVG;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.aggregators.Function.PostProcessingFunction.RATE;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.aggregators.Function.PostProcessingFunction.DIFF;
-import static org.assertj.core.api.Assertions.assertThat;
-
-public class FunctionTest {
-
- @Test
- public void testCreation() throws Exception {
- Function f = fromMetricName("Metric._avg");
- assertThat(f).isEqualTo(new Function(AVG, null));
-
- f = fromMetricName("Metric._rate._avg");
- assertThat(f).isEqualTo(new Function(AVG, RATE));
-
- f = fromMetricName("bytes_in");
- assertThat(f).isEqualTo(Function.DEFAULT_VALUE_FUNCTION);
-
- // Rate support without aggregates
- f = fromMetricName("Metric._rate");
- assertThat(f).isEqualTo(new Function(null, RATE));
-
- // Diff support
- f = fromMetricName("Metric._diff._avg");
- assertThat(f).isEqualTo(new Function(AVG, DIFF));
-
- // Diff support without aggregates
- f = fromMetricName("Metric._diff");
- assertThat(f).isEqualTo(new Function(null, DIFF));
-
- }
-
- @Ignore // If unknown function: behavior is best effort query without function
- @Test(expected = Function.FunctionFormatException.class)
- public void testNotAFunction() throws Exception {
- fromMetricName("bytes._not._afunction");
- }
-}
[26/28] ambari git commit: AMBARI-22740 : Fix integration test for
HBase in branch-3.0-ams due to UUID changes. (avijayan)
Posted by av...@apache.org.
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/PhoenixHBaseAccessor.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/PhoenixHBaseAccessor.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/PhoenixHBaseAccessor.java
new file mode 100644
index 0000000..325a039
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/PhoenixHBaseAccessor.java
@@ -0,0 +1,1844 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.metrics.core.timeline;
+
+import static java.util.concurrent.TimeUnit.SECONDS;
+
+import java.io.IOException;
+import java.sql.Connection;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.sql.Timestamp;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.TreeMap;
+import java.util.concurrent.ArrayBlockingQueue;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.ambari.metrics.core.timeline.aggregators.AggregatorUtils;
+import org.apache.ambari.metrics.core.timeline.aggregators.Function;
+import org.apache.ambari.metrics.core.timeline.aggregators.TimelineClusterMetric;
+import org.apache.ambari.metrics.core.timeline.aggregators.TimelineMetricReadHelper;
+import org.apache.ambari.metrics.core.timeline.discovery.TimelineMetricHostMetadata;
+import org.apache.ambari.metrics.core.timeline.discovery.TimelineMetricMetadataKey;
+import org.apache.ambari.metrics.core.timeline.discovery.TimelineMetricMetadataManager;
+import org.apache.ambari.metrics.core.timeline.query.Condition;
+import org.apache.ambari.metrics.core.timeline.query.DefaultPhoenixDataSource;
+import org.apache.ambari.metrics.core.timeline.query.PhoenixConnectionProvider;
+import org.apache.ambari.metrics.core.timeline.query.PhoenixTransactSQL;
+import org.apache.ambari.metrics.core.timeline.query.SplitByMetricNamesCondition;
+import org.apache.ambari.metrics.core.timeline.sink.ExternalMetricsSink;
+import org.apache.ambari.metrics.core.timeline.sink.ExternalSinkProvider;
+import org.apache.ambari.metrics.core.timeline.source.InternalMetricsSource;
+import org.apache.ambari.metrics.core.timeline.source.InternalSourceProvider;
+import org.apache.commons.lang.StringUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.DoNotRetryIOException;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
+import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
+import org.apache.hadoop.hbase.client.Durability;
+import org.apache.hadoop.hbase.client.TableDescriptor;
+import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
+import org.apache.hadoop.hbase.util.RetryCounter;
+import org.apache.hadoop.hbase.util.RetryCounterFactory;
+import org.apache.hadoop.metrics2.sink.timeline.ContainerMetric;
+import org.apache.hadoop.metrics2.sink.timeline.MetricClusterAggregate;
+import org.apache.hadoop.metrics2.sink.timeline.MetricHostAggregate;
+import org.apache.hadoop.metrics2.sink.timeline.Precision;
+import org.apache.hadoop.metrics2.sink.timeline.SingleValuedTimelineMetric;
+import org.apache.hadoop.metrics2.sink.timeline.TimelineMetric;
+import org.apache.hadoop.metrics2.sink.timeline.TimelineMetricMetadata;
+import org.apache.hadoop.metrics2.sink.timeline.TimelineMetrics;
+import org.apache.hadoop.util.ReflectionUtils;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.util.timeline.TimelineUtils;
+import org.apache.phoenix.exception.PhoenixIOException;
+import org.codehaus.jackson.map.ObjectMapper;
+import org.codehaus.jackson.type.TypeReference;
+
+import com.google.common.collect.Multimap;
+
+
+/**
+ * Provides a facade over the Phoenix API to access HBase schema
+ */
+public class PhoenixHBaseAccessor {
+ private static final Log LOG = LogFactory.getLog(PhoenixHBaseAccessor.class);
+
+ static final int PHOENIX_MAX_MUTATION_STATE_SIZE = 50000;
+ // Default stale data allowance set to 3 minutes, 2 minutes more than time
+ // it was collected. Also 2 minutes is the default aggregation interval at
+ // cluster and host levels.
+ static final long DEFAULT_OUT_OF_BAND_TIME_ALLOWANCE = 300000;
+ /**
+ * 22 metrics for 2hours in SECONDS (10 second data)
+ * => Reasonable upper bound on the limit such that our Precision calculation for a given time range makes sense.
+ */
+ private static final int METRICS_PER_MINUTE = 22;
+ private static final int POINTS_PER_MINUTE = 6;
+ public static int RESULTSET_LIMIT = (int)TimeUnit.HOURS.toMinutes(2) * METRICS_PER_MINUTE * POINTS_PER_MINUTE ;
+
+ static TimelineMetricReadHelper TIMELINE_METRIC_READ_HELPER = new TimelineMetricReadHelper();
+ static ObjectMapper mapper = new ObjectMapper();
+ static TypeReference<TreeMap<Long, Double>> metricValuesTypeRef = new TypeReference<TreeMap<Long, Double>>() {};
+
+ private final Configuration hbaseConf;
+ private final Configuration metricsConf;
+ private final RetryCounterFactory retryCounterFactory;
+ private final PhoenixConnectionProvider dataSource;
+ private final long outOfBandTimeAllowance;
+ private final int cacheSize;
+ private final boolean cacheEnabled;
+ private final BlockingQueue<TimelineMetrics> insertCache;
+ private ScheduledExecutorService scheduledExecutorService;
+ private MetricsCacheCommitterThread metricsCommiterThread;
+ private TimelineMetricsAggregatorSink aggregatorSink;
+ private final int cacheCommitInterval;
+ private final boolean skipBlockCacheForAggregatorsEnabled;
+ private final String timelineMetricsTablesDurability;
+ private final String timelineMetricsPrecisionTableDurability;
+ private TimelineMetricMetadataManager metadataManagerInstance;
+
+ static final String HSTORE_COMPACTION_CLASS_KEY =
+ "hbase.hstore.defaultengine.compactionpolicy.class";
+ static final String HSTORE_ENGINE_CLASS =
+ "hbase.hstore.engine.class";
+ static final String FIFO_COMPACTION_POLICY_CLASS =
+ "org.apache.hadoop.hbase.regionserver.compactions.FIFOCompactionPolicy";
+ static final String DATE_TIERED_COMPACTION_POLICY =
+ "org.apache.hadoop.hbase.regionserver.DateTieredStoreEngine";
+ static final String BLOCKING_STORE_FILES_KEY =
+ "hbase.hstore.blockingStoreFiles";
+
+ private Map<String, Integer> tableTTL = new HashMap<>();
+
+ private final TimelineMetricConfiguration configuration;
+ private List<InternalMetricsSource> rawMetricsSources = new ArrayList<>();
+
+ public PhoenixHBaseAccessor(PhoenixConnectionProvider dataSource) {
+ this(TimelineMetricConfiguration.getInstance(), dataSource);
+ }
+
+ // Test friendly construction since mock instrumentation is difficult to get
+ // working with hadoop mini cluster
+ PhoenixHBaseAccessor(TimelineMetricConfiguration configuration,
+ PhoenixConnectionProvider dataSource) {
+ this.configuration = TimelineMetricConfiguration.getInstance();
+ try {
+ this.hbaseConf = configuration.getHbaseConf();
+ this.metricsConf = configuration.getMetricsConf();
+ } catch (Exception e) {
+ throw new ExceptionInInitializerError("Cannot initialize configuration.");
+ }
+ if (dataSource == null) {
+ dataSource = new DefaultPhoenixDataSource(hbaseConf);
+ }
+ this.dataSource = dataSource;
+
+ RESULTSET_LIMIT = metricsConf.getInt(TimelineMetricConfiguration.GLOBAL_RESULT_LIMIT, RESULTSET_LIMIT);
+ try {
+ Class.forName("org.apache.phoenix.jdbc.PhoenixDriver");
+ } catch (ClassNotFoundException e) {
+ LOG.error("Phoenix client jar not found in the classpath.", e);
+ throw new IllegalStateException(e);
+ }
+
+ this.retryCounterFactory = new RetryCounterFactory(metricsConf.getInt(TimelineMetricConfiguration.GLOBAL_MAX_RETRIES, 10),
+ (int) SECONDS.toMillis(metricsConf.getInt(TimelineMetricConfiguration.GLOBAL_RETRY_INTERVAL, 3)));
+ this.outOfBandTimeAllowance = metricsConf.getLong(TimelineMetricConfiguration.OUT_OFF_BAND_DATA_TIME_ALLOWANCE,
+ DEFAULT_OUT_OF_BAND_TIME_ALLOWANCE);
+ this.cacheEnabled = Boolean.valueOf(metricsConf.get(TimelineMetricConfiguration.TIMELINE_METRICS_CACHE_ENABLED, "true"));
+ this.cacheSize = Integer.valueOf(metricsConf.get(TimelineMetricConfiguration.TIMELINE_METRICS_CACHE_SIZE, "150"));
+ this.cacheCommitInterval = Integer.valueOf(metricsConf.get(TimelineMetricConfiguration.TIMELINE_METRICS_CACHE_COMMIT_INTERVAL, "3"));
+ this.insertCache = new ArrayBlockingQueue<TimelineMetrics>(cacheSize);
+ this.skipBlockCacheForAggregatorsEnabled = metricsConf.getBoolean(TimelineMetricConfiguration.AGGREGATORS_SKIP_BLOCK_CACHE, false);
+ this.timelineMetricsTablesDurability = metricsConf.get(TimelineMetricConfiguration.TIMELINE_METRICS_AGGREGATE_TABLES_DURABILITY, "");
+ this.timelineMetricsPrecisionTableDurability = metricsConf.get(TimelineMetricConfiguration.TIMELINE_METRICS_PRECISION_TABLE_DURABILITY, "");
+
+ tableTTL.put(PhoenixTransactSQL.METRICS_RECORD_TABLE_NAME, metricsConf.getInt(TimelineMetricConfiguration.PRECISION_TABLE_TTL, 1 * 86400)); // 1 day
+ tableTTL.put(PhoenixTransactSQL.CONTAINER_METRICS_TABLE_NAME, metricsConf.getInt(TimelineMetricConfiguration.CONTAINER_METRICS_TTL, 30 * 86400)); // 30 days
+ tableTTL.put(PhoenixTransactSQL.METRICS_AGGREGATE_MINUTE_TABLE_NAME, metricsConf.getInt(TimelineMetricConfiguration.HOST_MINUTE_TABLE_TTL, 7 * 86400)); //7 days
+ tableTTL.put(PhoenixTransactSQL.METRICS_AGGREGATE_HOURLY_TABLE_NAME, metricsConf.getInt(TimelineMetricConfiguration.HOST_HOUR_TABLE_TTL, 30 * 86400)); //30 days
+ tableTTL.put(PhoenixTransactSQL.METRICS_AGGREGATE_DAILY_TABLE_NAME, metricsConf.getInt(TimelineMetricConfiguration.HOST_DAILY_TABLE_TTL, 365 * 86400)); //1 year
+ tableTTL.put(PhoenixTransactSQL.METRICS_CLUSTER_AGGREGATE_TABLE_NAME, metricsConf.getInt(TimelineMetricConfiguration.CLUSTER_SECOND_TABLE_TTL, 7 * 86400)); //7 days
+ tableTTL.put(PhoenixTransactSQL.METRICS_CLUSTER_AGGREGATE_MINUTE_TABLE_NAME, metricsConf.getInt(TimelineMetricConfiguration.CLUSTER_MINUTE_TABLE_TTL, 30 * 86400)); //30 days
+ tableTTL.put(PhoenixTransactSQL.METRICS_CLUSTER_AGGREGATE_HOURLY_TABLE_NAME, metricsConf.getInt(TimelineMetricConfiguration.CLUSTER_HOUR_TABLE_TTL, 365 * 86400)); //1 year
+ tableTTL.put(PhoenixTransactSQL.METRICS_CLUSTER_AGGREGATE_DAILY_TABLE_NAME, metricsConf.getInt(TimelineMetricConfiguration.CLUSTER_DAILY_TABLE_TTL, 730 * 86400)); //2 years
+
+ if (cacheEnabled) {
+ LOG.debug("Initialising and starting metrics cache committer thread...");
+ metricsCommiterThread = new MetricsCacheCommitterThread(this);
+ scheduledExecutorService = Executors.newSingleThreadScheduledExecutor();
+ scheduledExecutorService.scheduleWithFixedDelay(metricsCommiterThread, 0, cacheCommitInterval, TimeUnit.SECONDS);
+ }
+
+ Class<? extends TimelineMetricsAggregatorSink> metricSinkClass =
+ metricsConf.getClass(TimelineMetricConfiguration.TIMELINE_METRIC_AGGREGATOR_SINK_CLASS, null,
+ TimelineMetricsAggregatorSink.class);
+ if (metricSinkClass != null) {
+ aggregatorSink = ReflectionUtils.newInstance(metricSinkClass, metricsConf);
+ LOG.info("Initialized aggregator sink class " + metricSinkClass);
+ }
+
+ List<ExternalSinkProvider> externalSinkProviderList = configuration.getExternalSinkProviderList();
+ InternalSourceProvider internalSourceProvider = configuration.getInternalSourceProvider();
+ if (!externalSinkProviderList.isEmpty()) {
+ for (ExternalSinkProvider externalSinkProvider : externalSinkProviderList) {
+ ExternalMetricsSink rawMetricsSink = externalSinkProvider.getExternalMetricsSink(InternalSourceProvider.SOURCE_NAME.RAW_METRICS);
+ int interval = configuration.getExternalSinkInterval(externalSinkProvider.getClass().getSimpleName(), InternalSourceProvider.SOURCE_NAME.RAW_METRICS);
+ if (interval == -1) {
+ interval = cacheCommitInterval;
+ }
+ rawMetricsSources.add(internalSourceProvider.getInternalMetricsSource(InternalSourceProvider.SOURCE_NAME.RAW_METRICS, interval, rawMetricsSink));
+ }
+ }
+ TIMELINE_METRIC_READ_HELPER = new TimelineMetricReadHelper(this.metadataManagerInstance);
+ }
+
+ public boolean isInsertCacheEmpty() {
+ return insertCache.isEmpty();
+ }
+
+ public void commitMetricsFromCache() {
+ LOG.debug("Clearing metrics cache");
+ List<TimelineMetrics> metricsList = new ArrayList<TimelineMetrics>(insertCache.size());
+ if (!insertCache.isEmpty()) {
+ insertCache.drainTo(metricsList); // More performant than poll
+ }
+ if (metricsList.size() > 0) {
+ commitMetrics(metricsList);
+ if (!rawMetricsSources.isEmpty()) {
+ for (InternalMetricsSource rawMetricsSource : rawMetricsSources) {
+ rawMetricsSource.publishTimelineMetrics(metricsList);
+ }
+ }
+ }
+ }
+
+ public void commitMetrics(TimelineMetrics timelineMetrics) {
+ commitMetrics(Collections.singletonList(timelineMetrics));
+ }
+
+ public void commitMetrics(Collection<TimelineMetrics> timelineMetricsCollection) {
+ LOG.debug("Committing metrics to store");
+ Connection conn = null;
+ PreparedStatement metricRecordStmt = null;
+
+ try {
+ conn = getConnection();
+ metricRecordStmt = conn.prepareStatement(String.format(
+ PhoenixTransactSQL.UPSERT_METRICS_SQL, PhoenixTransactSQL.METRICS_RECORD_TABLE_NAME));
+ for (TimelineMetrics timelineMetrics : timelineMetricsCollection) {
+ for (TimelineMetric metric : timelineMetrics.getMetrics()) {
+
+ metricRecordStmt.clearParameters();
+
+ if (LOG.isTraceEnabled()) {
+ LOG.trace("host: " + metric.getHostName() + ", " +
+ "metricName = " + metric.getMetricName() + ", " +
+ "values: " + metric.getMetricValues());
+ }
+ double[] aggregates = AggregatorUtils.calculateAggregates(
+ metric.getMetricValues());
+
+ byte[] uuid = metadataManagerInstance.getUuid(metric);
+ if (uuid == null) {
+ LOG.error("Error computing UUID for metric. Cannot write metrics : " + metric.toString());
+ continue;
+ }
+ metricRecordStmt.setBytes(1, uuid);
+ metricRecordStmt.setLong(2, metric.getStartTime());
+ metricRecordStmt.setDouble(3, aggregates[0]);
+ metricRecordStmt.setDouble(4, aggregates[1]);
+ metricRecordStmt.setDouble(5, aggregates[2]);
+ metricRecordStmt.setInt(6, (int) aggregates[3]);
+ String json = TimelineUtils.dumpTimelineRecordtoJSON(metric.getMetricValues());
+ metricRecordStmt.setString(7, json);
+
+ try {
+ metricRecordStmt.executeUpdate();
+ } catch (SQLException sql) {
+ LOG.error("Failed on insert records to store.", sql);
+ }
+ }
+ }
+
+ // commit() blocked if HBase unavailable
+ conn.commit();
+ } catch (Exception exception){
+ exception.printStackTrace();
+ }
+ finally {
+ if (metricRecordStmt != null) {
+ try {
+ metricRecordStmt.close();
+ } catch (SQLException e) {
+ // Ignore
+ }
+ }
+ if (conn != null) {
+ try {
+ conn.close();
+ } catch (SQLException sql) {
+ // Ignore
+ }
+ }
+ }
+ }
+
+ private static TimelineMetric getLastTimelineMetricFromResultSet(ResultSet rs)
+ throws SQLException, IOException {
+ TimelineMetric metric = TIMELINE_METRIC_READ_HELPER.getTimelineMetricCommonsFromResultSet(rs);
+ metric.setMetricValues(readLastMetricValueFromJSON(rs.getString("METRICS")));
+ return metric;
+ }
+
+ private static TreeMap<Long, Double> readLastMetricValueFromJSON(String json)
+ throws IOException {
+ TreeMap<Long, Double> values = readMetricFromJSON(json);
+ Long lastTimeStamp = values.lastKey();
+
+ TreeMap<Long, Double> valueMap = new TreeMap<Long, Double>();
+ valueMap.put(lastTimeStamp, values.get(lastTimeStamp));
+ return valueMap;
+ }
+
+ @SuppressWarnings("unchecked")
+ public static TreeMap<Long, Double> readMetricFromJSON(String json) throws IOException {
+ return mapper.readValue(json, metricValuesTypeRef);
+ }
+
+ private Connection getConnectionRetryingOnException()
+ throws SQLException, InterruptedException {
+ RetryCounter retryCounter = retryCounterFactory.create();
+ while (true) {
+ try{
+ return getConnection();
+ } catch (SQLException e) {
+ if(!retryCounter.shouldRetry()){
+ LOG.error("HBaseAccessor getConnection failed after "
+ + retryCounter.getMaxAttempts() + " attempts");
+ throw e;
+ }
+ }
+ retryCounter.sleepUntilNextRetry();
+ }
+ }
+
+ /**
+ * Get JDBC connection to HBase store. Assumption is that the hbase
+ * configuration is present on the classpath and loaded by the caller into
+ * the Configuration object.
+ * Phoenix already caches the HConnection between the client and HBase
+ * cluster.
+ *
+ * @return @java.sql.Connection
+ */
+ public Connection getConnection() throws SQLException {
+ return dataSource.getConnection();
+ }
+
+ /**
+ * Unit test purpose only for now.
+ * @return @HBaseAdmin
+ * @throws IOException
+ */
+ Admin getHBaseAdmin() throws IOException {
+ return dataSource.getHBaseAdmin();
+ }
+
+ protected void initMetricSchema() {
+ Connection conn = null;
+ Statement stmt = null;
+
+ String encoding = metricsConf.get(TimelineMetricConfiguration.HBASE_ENCODING_SCHEME, PhoenixTransactSQL.DEFAULT_ENCODING);
+ String compression = metricsConf.get(TimelineMetricConfiguration.HBASE_COMPRESSION_SCHEME, PhoenixTransactSQL.DEFAULT_TABLE_COMPRESSION);
+
+
+ try {
+ LOG.info("Initializing metrics schema...");
+ conn = getConnectionRetryingOnException();
+ stmt = conn.createStatement();
+
+ // Metadata
+ String metadataSql = String.format(PhoenixTransactSQL.CREATE_METRICS_METADATA_TABLE_SQL,
+ encoding, compression);
+ stmt.executeUpdate(metadataSql);
+ stmt.executeUpdate(PhoenixTransactSQL.ALTER_METRICS_METADATA_TABLE);
+
+ String hostedAppSql = String.format(PhoenixTransactSQL.CREATE_HOSTED_APPS_METADATA_TABLE_SQL,
+ encoding, compression);
+ stmt.executeUpdate(hostedAppSql);
+
+ //Host Instances table
+ String hostedInstancesSql = String.format(PhoenixTransactSQL.CREATE_INSTANCE_HOST_TABLE_SQL,
+ encoding, compression);
+ stmt.executeUpdate(hostedInstancesSql);
+
+ // Container Metrics
+ stmt.executeUpdate( String.format(PhoenixTransactSQL.CREATE_CONTAINER_METRICS_TABLE_SQL,
+ encoding, tableTTL.get(PhoenixTransactSQL.CONTAINER_METRICS_TABLE_NAME), compression));
+
+ // Host level
+ String precisionSql = String.format(PhoenixTransactSQL.CREATE_METRICS_TABLE_SQL,
+ encoding, tableTTL.get(PhoenixTransactSQL.METRICS_RECORD_TABLE_NAME), compression);
+ stmt.executeUpdate(precisionSql);
+
+ String hostMinuteAggregrateSql = String.format(PhoenixTransactSQL.CREATE_METRICS_AGGREGATE_TABLE_SQL,
+ PhoenixTransactSQL.METRICS_AGGREGATE_MINUTE_TABLE_NAME, encoding,
+ tableTTL.get(PhoenixTransactSQL.METRICS_AGGREGATE_MINUTE_TABLE_NAME),
+ compression);
+ stmt.executeUpdate(hostMinuteAggregrateSql);
+
+ stmt.executeUpdate(String.format(PhoenixTransactSQL.CREATE_METRICS_AGGREGATE_TABLE_SQL,
+ PhoenixTransactSQL.METRICS_AGGREGATE_HOURLY_TABLE_NAME, encoding,
+ tableTTL.get(PhoenixTransactSQL.METRICS_AGGREGATE_HOURLY_TABLE_NAME),
+ compression));
+ stmt.executeUpdate(String.format(PhoenixTransactSQL.CREATE_METRICS_AGGREGATE_TABLE_SQL,
+ PhoenixTransactSQL.METRICS_AGGREGATE_DAILY_TABLE_NAME, encoding,
+ tableTTL.get(PhoenixTransactSQL.METRICS_AGGREGATE_DAILY_TABLE_NAME),
+ compression));
+
+ // Cluster level
+ String aggregateSql = String.format(PhoenixTransactSQL.CREATE_METRICS_CLUSTER_AGGREGATE_TABLE_SQL,
+ PhoenixTransactSQL.METRICS_CLUSTER_AGGREGATE_TABLE_NAME, encoding,
+ tableTTL.get(PhoenixTransactSQL.METRICS_CLUSTER_AGGREGATE_TABLE_NAME),
+ compression);
+
+ stmt.executeUpdate(aggregateSql);
+ stmt.executeUpdate(String.format(PhoenixTransactSQL.CREATE_METRICS_CLUSTER_AGGREGATE_GROUPED_TABLE_SQL,
+ PhoenixTransactSQL.METRICS_CLUSTER_AGGREGATE_MINUTE_TABLE_NAME, encoding,
+ tableTTL.get(PhoenixTransactSQL.METRICS_CLUSTER_AGGREGATE_MINUTE_TABLE_NAME),
+ compression));
+ stmt.executeUpdate(String.format(PhoenixTransactSQL.CREATE_METRICS_CLUSTER_AGGREGATE_GROUPED_TABLE_SQL,
+ PhoenixTransactSQL.METRICS_CLUSTER_AGGREGATE_HOURLY_TABLE_NAME, encoding,
+ tableTTL.get(PhoenixTransactSQL.METRICS_CLUSTER_AGGREGATE_HOURLY_TABLE_NAME),
+ compression));
+ stmt.executeUpdate(String.format(PhoenixTransactSQL.CREATE_METRICS_CLUSTER_AGGREGATE_GROUPED_TABLE_SQL,
+ PhoenixTransactSQL.METRICS_CLUSTER_AGGREGATE_DAILY_TABLE_NAME, encoding,
+ tableTTL.get(PhoenixTransactSQL.METRICS_CLUSTER_AGGREGATE_DAILY_TABLE_NAME),
+ compression));
+
+
+ conn.commit();
+
+ LOG.info("Metrics schema initialized.");
+ } catch (SQLException | InterruptedException sql) {
+ LOG.error("Error creating Metrics Schema in HBase using Phoenix.", sql);
+ throw new MetricsSystemInitializationException(
+ "Error creating Metrics Schema in HBase using Phoenix.", sql);
+ } finally {
+ if (stmt != null) {
+ try {
+ stmt.close();
+ } catch (SQLException e) {
+ // Ignore
+ }
+ }
+ if (conn != null) {
+ try {
+ conn.close();
+ } catch (SQLException e) {
+ // Ignore
+ }
+ }
+ }
+ }
+
+ protected void initPoliciesAndTTL() {
+ Admin hBaseAdmin = null;
+ try {
+ hBaseAdmin = dataSource.getHBaseAdmin();
+ } catch (IOException e) {
+ LOG.warn("Unable to initialize HBaseAdmin for setting policies.", e);
+ }
+
+ TableName[] tableNames = null;
+ if (hBaseAdmin != null) {
+ try {
+ tableNames = hBaseAdmin.listTableNames(PhoenixTransactSQL.PHOENIX_TABLES_REGEX_PATTERN_STRING, false);
+ } catch (IOException e) {
+ LOG.warn("Unable to get table names from HBaseAdmin for setting policies.", e);
+ return;
+ }
+ if (tableNames == null || tableNames.length == 0) {
+ LOG.warn("Unable to get table names from HBaseAdmin for setting policies.");
+ return;
+ }
+ for (String tableName : PhoenixTransactSQL.PHOENIX_TABLES) {
+ try {
+ boolean modifyTable = false;
+ Optional<TableName> tableNameOptional = Arrays.stream(tableNames)
+ .filter(t -> tableName.equals(t.getNameAsString())).findFirst();
+
+ TableDescriptor tableDescriptor = null;
+ if (tableNameOptional.isPresent()) {
+ tableDescriptor = hBaseAdmin.getTableDescriptor(tableNameOptional.get());
+ }
+
+ if (tableDescriptor == null) {
+ LOG.warn("Unable to get table descriptor for " + tableName);
+ continue;
+ }
+
+ // @TableDescriptor is immutable by design
+ TableDescriptorBuilder tableDescriptorBuilder =
+ TableDescriptorBuilder.newBuilder(tableDescriptor);
+
+ //Set normalizer preferences
+ boolean enableNormalizer = hbaseConf.getBoolean("hbase.normalizer.enabled", false);
+ if (enableNormalizer ^ tableDescriptor.isNormalizationEnabled()) {
+ tableDescriptorBuilder.setNormalizationEnabled(enableNormalizer);
+ LOG.info("Normalizer set to " + enableNormalizer + " for " + tableName);
+ modifyTable = true;
+ }
+
+ //Set durability preferences
+ boolean durabilitySettingsModified = setDurabilityForTable(tableName, tableDescriptorBuilder);
+ modifyTable = modifyTable || durabilitySettingsModified;
+
+ //Set compaction policy preferences
+ boolean compactionPolicyModified = false;
+ compactionPolicyModified = setCompactionPolicyForTable(tableName, tableDescriptorBuilder);
+ modifyTable = modifyTable || compactionPolicyModified;
+
+ // Change TTL setting to match user configuration
+ ColumnFamilyDescriptor[] columnFamilyDescriptors = tableDescriptor.getColumnFamilies();
+ if (columnFamilyDescriptors != null) {
+ for (ColumnFamilyDescriptor familyDescriptor : columnFamilyDescriptors) {
+ int ttlValue = familyDescriptor.getTimeToLive();
+ if (ttlValue != tableTTL.get(tableName)) {
+ ColumnFamilyDescriptorBuilder familyDescriptorBuilder =
+ ColumnFamilyDescriptorBuilder.newBuilder(familyDescriptor);
+
+ familyDescriptorBuilder.setTimeToLive(tableTTL.get(tableName));
+
+ LOG.info("Setting TTL on table: " + tableName + " to : " +
+ tableTTL.get(tableName) + " seconds.");
+
+ hBaseAdmin.modifyColumnFamily(tableNameOptional.get(), familyDescriptorBuilder.build());
+ // modifyTable = true;
+ }
+ }
+ }
+
+ // Persist only if anything changed
+ if (modifyTable) {
+ hBaseAdmin.modifyTable(tableNameOptional.get(), tableDescriptorBuilder.build());
+ }
+
+ } catch (IOException e) {
+ LOG.error("Failed setting policies for " + tableName, e);
+ }
+ }
+ try {
+ hBaseAdmin.close();
+ } catch (IOException e) {
+ LOG.warn("Exception on HBaseAdmin close.", e);
+ }
+ }
+ }
+
+ private boolean setDurabilityForTable(String tableName, TableDescriptorBuilder tableDescriptor) {
+
+ boolean modifyTable = false;
+ // Set WAL preferences
+ if (PhoenixTransactSQL.METRICS_RECORD_TABLE_NAME.equals(tableName)) {
+ if (!timelineMetricsPrecisionTableDurability.isEmpty()) {
+ LOG.info("Setting WAL option " + timelineMetricsPrecisionTableDurability + " for table : " + tableName);
+ boolean validDurability = true;
+ if ("SKIP_WAL".equals(timelineMetricsPrecisionTableDurability)) {
+ tableDescriptor.setDurability(Durability.SKIP_WAL);
+ } else if ("SYNC_WAL".equals(timelineMetricsPrecisionTableDurability)) {
+ tableDescriptor.setDurability(Durability.SYNC_WAL);
+ } else if ("ASYNC_WAL".equals(timelineMetricsPrecisionTableDurability)) {
+ tableDescriptor.setDurability(Durability.ASYNC_WAL);
+ } else if ("FSYNC_WAL".equals(timelineMetricsPrecisionTableDurability)) {
+ tableDescriptor.setDurability(Durability.FSYNC_WAL);
+ } else {
+ LOG.info("Unknown value for " + TimelineMetricConfiguration.TIMELINE_METRICS_PRECISION_TABLE_DURABILITY + " : " + timelineMetricsPrecisionTableDurability);
+ validDurability = false;
+ }
+ if (validDurability) {
+ modifyTable = true;
+ }
+ }
+ } else {
+ if (!timelineMetricsTablesDurability.isEmpty()) {
+ LOG.info("Setting WAL option " + timelineMetricsTablesDurability + " for table : " + tableName);
+ boolean validDurability = true;
+ if ("SKIP_WAL".equals(timelineMetricsTablesDurability)) {
+ tableDescriptor.setDurability(Durability.SKIP_WAL);
+ } else if ("SYNC_WAL".equals(timelineMetricsTablesDurability)) {
+ tableDescriptor.setDurability(Durability.SYNC_WAL);
+ } else if ("ASYNC_WAL".equals(timelineMetricsTablesDurability)) {
+ tableDescriptor.setDurability(Durability.ASYNC_WAL);
+ } else if ("FSYNC_WAL".equals(timelineMetricsTablesDurability)) {
+ tableDescriptor.setDurability(Durability.FSYNC_WAL);
+ } else {
+ LOG.info("Unknown value for " + TimelineMetricConfiguration.TIMELINE_METRICS_AGGREGATE_TABLES_DURABILITY + " : " + timelineMetricsTablesDurability);
+ validDurability = false;
+ }
+ if (validDurability) {
+ modifyTable = true;
+ }
+ }
+ }
+ return modifyTable;
+ }
+
+ private boolean setCompactionPolicyForTable(String tableName, TableDescriptorBuilder tableDescriptorBuilder) {
+
+ boolean modifyTable = false;
+
+ String compactionPolicyKey = metricsConf.get(TimelineMetricConfiguration.TIMELINE_METRICS_HBASE_AGGREGATE_TABLE_COMPACTION_POLICY_KEY,
+ HSTORE_ENGINE_CLASS);
+ String compactionPolicyClass = metricsConf.get(TimelineMetricConfiguration.TIMELINE_METRICS_HBASE_AGGREGATE_TABLE_COMPACTION_POLICY_CLASS,
+ DATE_TIERED_COMPACTION_POLICY);
+ int blockingStoreFiles = hbaseConf.getInt(TimelineMetricConfiguration.TIMELINE_METRICS_AGGREGATE_TABLE_HBASE_BLOCKING_STORE_FILES, 60);
+
+ if (tableName.equals(PhoenixTransactSQL.METRICS_RECORD_TABLE_NAME)) {
+ compactionPolicyKey = metricsConf.get(TimelineMetricConfiguration.TIMELINE_METRICS_HBASE_PRECISION_TABLE_COMPACTION_POLICY_KEY,
+ HSTORE_COMPACTION_CLASS_KEY);
+ compactionPolicyClass = metricsConf.get(TimelineMetricConfiguration.TIMELINE_METRICS_HBASE_PRECISION_TABLE_COMPACTION_POLICY_CLASS,
+ FIFO_COMPACTION_POLICY_CLASS);
+ blockingStoreFiles = hbaseConf.getInt(TimelineMetricConfiguration.TIMELINE_METRICS_PRECISION_TABLE_HBASE_BLOCKING_STORE_FILES, 1000);
+ }
+
+ if (StringUtils.isEmpty(compactionPolicyKey) || StringUtils.isEmpty(compactionPolicyClass)) {
+ // Default blockingStoreFiles = 300
+ modifyTable = setHbaseBlockingStoreFiles(tableDescriptorBuilder, tableName, 300);
+ } else {
+ tableDescriptorBuilder.setValue(compactionPolicyKey, compactionPolicyClass);
+ if (tableName.equals(PhoenixTransactSQL.METRICS_RECORD_TABLE_NAME)) {
+ tableDescriptorBuilder.removeValue(HSTORE_ENGINE_CLASS.getBytes());
+ } else {
+ tableDescriptorBuilder.removeValue(HSTORE_COMPACTION_CLASS_KEY.getBytes());
+ }
+ setHbaseBlockingStoreFiles(tableDescriptorBuilder, tableName, blockingStoreFiles);
+ modifyTable = true;
+ }
+
+ return modifyTable;
+ }
+
+ private boolean setHbaseBlockingStoreFiles(TableDescriptorBuilder tableDescriptor,
+ String tableName, int value) {
+ tableDescriptor.setValue(BLOCKING_STORE_FILES_KEY, String.valueOf(value));
+ LOG.info("Setting config property " + BLOCKING_STORE_FILES_KEY +
+ " = " + value + " for " + tableName);
+ return true;
+ }
+
+ protected String getSplitPointsStr(String splitPoints) {
+ if (StringUtils.isEmpty(splitPoints.trim())) {
+ return "";
+ }
+ String[] points = splitPoints.split(",");
+ if (points.length > 0) {
+ StringBuilder sb = new StringBuilder(" SPLIT ON ");
+ sb.append("(");
+ for (String point : points) {
+ sb.append("'");
+ sb.append(point.trim());
+ sb.append("'");
+ sb.append(",");
+ }
+ sb.deleteCharAt(sb.length() - 1);
+ sb.append(")");
+ return sb.toString();
+ }
+ return "";
+ }
+
+ /**
+ * Insert precision YARN container data.
+ */
+ public void insertContainerMetrics(List<ContainerMetric> metrics)
+ throws SQLException, IOException {
+ Connection conn = getConnection();
+ PreparedStatement metricRecordStmt = null;
+
+ try {
+ metricRecordStmt = conn.prepareStatement(
+ String.format(PhoenixTransactSQL.UPSERT_CONTAINER_METRICS_SQL, PhoenixTransactSQL.CONTAINER_METRICS_TABLE_NAME));
+ for (ContainerMetric metric : metrics) {
+ metricRecordStmt.clearParameters();
+ metricRecordStmt.setString(1, ContainerId.fromString(metric.getContainerId())
+ .getApplicationAttemptId().getApplicationId().toString());
+ metricRecordStmt.setString(2, metric.getContainerId());
+ metricRecordStmt.setTimestamp(3, new Timestamp(metric.getStartTime()));
+ metricRecordStmt.setTimestamp(4, new Timestamp(metric.getFinishTime()));
+ metricRecordStmt.setLong(5, metric.getFinishTime() - metric.getStartTime());
+ metricRecordStmt.setString(6, metric.getHostName());
+ metricRecordStmt.setInt(7, metric.getExitCode());
+ metricRecordStmt.setLong(8, metric.getLocalizationDuration());
+ metricRecordStmt.setLong(9, metric.getLaunchDuration());
+ metricRecordStmt.setDouble(10, (double) metric.getPmemLimit() / 1024);
+ metricRecordStmt.setDouble(11,
+ ((double) metric.getPmemLimit() / 1024) * (metric.getFinishTime()
+ - metric.getStartTime()));
+ metricRecordStmt.setDouble(12, (double) metric.getVmemLimit() / 1024);
+ metricRecordStmt.setDouble(13, (double) metric.getPmemUsedMin() / 1024);
+ metricRecordStmt.setDouble(14, (double) metric.getPmemUsedMax() / 1024);
+ metricRecordStmt.setDouble(15, (double) metric.getPmemUsedAvg() / 1024);
+ metricRecordStmt.setDouble(16, (double) metric.getPmem50Pct() / 1024);
+ metricRecordStmt.setDouble(17, (double) metric.getPmem75Pct() / 1024);
+ metricRecordStmt.setDouble(18, (double) metric.getPmem90Pct() / 1024);
+ metricRecordStmt.setDouble(19, (double) metric.getPmem95Pct()/ 1024);
+ metricRecordStmt.setDouble(20, (double) metric.getPmem99Pct() / 1024);
+ metricRecordStmt.setDouble(21, (double) metric.getPmemLimit() / 1024
+ - (double) metric.getPmemUsedMax() / 1024);
+ metricRecordStmt.setDouble(22, ((double) metric.getPmemLimit() / 1024
+ - (double) metric.getPmemUsedMax() / 1024) * (metric.getFinishTime()
+ - metric.getStartTime()));
+
+ try {
+ metricRecordStmt.executeUpdate();
+ } catch (SQLException sql) {
+ LOG.error("Failed on insert records to store.", sql);
+ }
+ }
+
+ conn.commit();
+ } finally {
+ if (metricRecordStmt != null) {
+ try {
+ metricRecordStmt.close();
+ } catch (SQLException e) {
+ // Ignore
+ }
+ }
+ if (conn != null) {
+ try {
+ conn.close();
+ } catch (SQLException sql) {
+ // Ignore
+ }
+ }
+ }
+ }
+
+ /**
+ * Insert precision data.
+ */
+ public void insertMetricRecordsWithMetadata(TimelineMetricMetadataManager metadataManager,
+ TimelineMetrics metrics, boolean skipCache) throws SQLException, IOException {
+ List<TimelineMetric> timelineMetrics = metrics.getMetrics();
+ if (timelineMetrics == null || timelineMetrics.isEmpty()) {
+ LOG.debug("Empty metrics insert request.");
+ return;
+ }
+ for (Iterator<TimelineMetric> iterator = timelineMetrics.iterator(); iterator.hasNext();) {
+
+ TimelineMetric tm = iterator.next();
+
+ boolean acceptMetric = TimelineMetricsFilter.acceptMetric(tm);
+
+ // Write to metadata cache on successful write to store
+ if (metadataManager != null) {
+ metadataManager.putIfModifiedTimelineMetricMetadata(
+ metadataManager.getTimelineMetricMetadata(tm, acceptMetric));
+
+ metadataManager.putIfModifiedHostedAppsMetadata(
+ tm.getHostName(), tm.getAppId());
+
+ if (!tm.getAppId().equals("FLUME_HANDLER")) {
+ metadataManager.putIfModifiedHostedInstanceMetadata(tm.getInstanceId(), tm.getHostName());
+ }
+ }
+ if (!acceptMetric) {
+ iterator.remove();
+ }
+ }
+
+ if (!skipCache && cacheEnabled) {
+ LOG.debug("Adding metrics to cache");
+ if (insertCache.size() >= cacheSize) {
+ commitMetricsFromCache();
+ }
+ try {
+ insertCache.put(metrics); // blocked while the queue is full
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ }
+ } else {
+ LOG.debug("Skipping metrics cache");
+ commitMetrics(metrics);
+ }
+ }
+
+ public void insertMetricRecords(TimelineMetrics metrics, boolean skipCache) throws SQLException, IOException {
+ insertMetricRecordsWithMetadata(null, metrics, skipCache);
+ }
+
+ public void insertMetricRecords(TimelineMetrics metrics) throws SQLException, IOException {
+ insertMetricRecords(metrics, false);
+ }
+
+
+ @SuppressWarnings("unchecked")
+ public TimelineMetrics getMetricRecords(
+ final Condition condition, Multimap<String, List<Function>> metricFunctions)
+ throws SQLException, IOException {
+
+ validateConditionIsNotEmpty(condition);
+
+ Connection conn = getConnection();
+ PreparedStatement stmt = null;
+ ResultSet rs = null;
+ TimelineMetrics metrics = new TimelineMetrics();
+
+ try {
+ //get latest
+ if (condition.isPointInTime()){
+ getLatestMetricRecords(condition, conn, metrics);
+ } else {
+ if (condition.getEndTime() >= condition.getStartTime()) {
+ stmt = PhoenixTransactSQL.prepareGetMetricsSqlStmt(conn, condition);
+ rs = stmt.executeQuery();
+ while (rs.next()) {
+ appendMetricFromResultSet(metrics, condition, metricFunctions, rs);
+ }
+ } else {
+ LOG.warn("Skipping metrics query because endTime < startTime");
+ }
+ }
+
+ } catch (PhoenixIOException pioe) {
+ Throwable pioe2 = pioe.getCause();
+ // Need to find out if this is exception "Could not find hash cache
+ // for joinId" or another PhoenixIOException
+ if (pioe2 instanceof PhoenixIOException &&
+ pioe2.getCause() instanceof DoNotRetryIOException) {
+ String className = null;
+ for (StackTraceElement ste : pioe2.getCause().getStackTrace()) {
+ className = ste.getClassName();
+ }
+
+ if (className != null && className.equals("HashJoinRegionScanner")) {
+ LOG.error("The cache might have expired and have been removed. Try to" +
+ " increase the cache size by setting bigger value for " +
+ "phoenix.coprocessor.maxMetaDataCacheSize in ams-hbase-site config." +
+ " Falling back to sort-merge join algorithm.");
+ PhoenixTransactSQL.setSortMergeJoinEnabled(true);
+ }
+ }
+ throw pioe;
+ } catch (RuntimeException ex) {
+ // We need to find out if this is a real IO exception
+ // or exception "maxStamp is smaller than minStamp"
+ // which is thrown in hbase TimeRange.java
+ Throwable io = ex.getCause();
+ String className = null;
+ if (io != null) {
+ for (StackTraceElement ste : io.getStackTrace()) {
+ className = ste.getClassName();
+ }
+ }
+ if (className != null && className.equals("TimeRange")) {
+ // This is "maxStamp is smaller than minStamp" exception
+ // Log error and return empty metrics
+ LOG.debug(io);
+ return new TimelineMetrics();
+ } else {
+ throw ex;
+ }
+
+ } finally {
+ if (rs != null) {
+ try {
+ rs.close();
+ } catch (SQLException e) {
+ // Ignore
+ }
+ }
+ if (stmt != null) {
+ try {
+ stmt.close();
+ } catch (SQLException e) {
+ // Ignore
+ }
+ }
+ if (conn != null) {
+ try {
+ conn.close();
+ } catch (SQLException sql) {
+ // Ignore
+ }
+ }
+ }
+
+ LOG.debug("Metrics records size: " + metrics.getMetrics().size());
+ return metrics;
+ }
+
+ /**
+ * Apply aggregate function to the result if supplied else get precision
+ * or aggregate data with default function applied.
+ */
+ private void appendMetricFromResultSet(TimelineMetrics metrics, Condition condition,
+ Multimap<String, List<Function>> metricFunctions,
+ ResultSet rs) throws SQLException, IOException {
+ byte[] uuid = rs.getBytes("UUID");
+ String metricName = metadataManagerInstance.getMetricNameFromUuid(uuid);
+ Collection<List<Function>> functionList = findMetricFunctions(metricFunctions, metricName);
+
+ for (List<Function> functions : functionList) {
+ // Apply aggregation function if present
+ if ((functions != null && !functions.isEmpty())) {
+ if (functions.size() > 1) {
+ throw new IllegalArgumentException("Multiple aggregate functions not supported.");
+ }
+ for (Function f : functions) {
+ if (f.getReadFunction() == Function.ReadFunction.VALUE) {
+ getTimelineMetricsFromResultSet(metrics, f, condition, rs);
+ } else {
+ SingleValuedTimelineMetric metric =
+ TIMELINE_METRIC_READ_HELPER.getAggregatedTimelineMetricFromResultSet(rs, f);
+
+ if (condition.isGrouped()) {
+ metrics.addOrMergeTimelineMetric(metric);
+ } else {
+ metrics.getMetrics().add(metric.getTimelineMetric());
+ }
+ }
+ }
+ } else {
+ // No aggregation requested
+ // Execution never goes here, function always contain at least 1 element
+ getTimelineMetricsFromResultSet(metrics, null, condition, rs);
+ }
+ }
+ }
+
+ private void getTimelineMetricsFromResultSet(TimelineMetrics metrics, Function f, Condition condition, ResultSet rs) throws SQLException, IOException {
+ if (condition.getPrecision().equals(Precision.SECONDS)) {
+ TimelineMetric metric = TIMELINE_METRIC_READ_HELPER.getTimelineMetricFromResultSet(rs);
+ if (f != null && f.getSuffix() != null) { //Case : Requesting "._rate" for precision data
+ metric.setMetricName(metric.getMetricName() + f.getSuffix());
+ }
+ if (condition.isGrouped()) {
+ metrics.addOrMergeTimelineMetric(metric);
+ } else {
+ metrics.getMetrics().add(metric);
+ }
+
+ } else {
+ SingleValuedTimelineMetric metric =
+ TIMELINE_METRIC_READ_HELPER.getAggregatedTimelineMetricFromResultSet(rs, f);
+ if (condition.isGrouped()) {
+ metrics.addOrMergeTimelineMetric(metric);
+ } else {
+ metrics.getMetrics().add(metric.getTimelineMetric());
+ }
+ }
+ }
+
+ private void getLatestMetricRecords(Condition condition, Connection conn,
+ TimelineMetrics metrics) throws SQLException, IOException {
+
+ validateConditionIsNotEmpty(condition);
+
+ PreparedStatement stmt = PhoenixTransactSQL.prepareGetLatestMetricSqlStmt(conn, condition);
+ ResultSet rs = null;
+ try {
+ rs = stmt.executeQuery();
+ while (rs.next()) {
+ TimelineMetric metric = getLastTimelineMetricFromResultSet(rs);
+ metrics.getMetrics().add(metric);
+ }
+ } finally {
+ if (rs != null) {
+ try {
+ rs.close();
+ } catch (SQLException e) {
+ // Ignore
+ }
+ }
+ if (stmt != null) {
+ stmt.close();
+ }
+ }
+ }
+
+ /**
+ * Get metrics aggregated across hosts.
+ *
+ * @param condition @Condition
+ * @return @TimelineMetrics
+ * @throws SQLException
+ */
+ public TimelineMetrics getAggregateMetricRecords(final Condition condition,
+ Multimap<String, List<Function>> metricFunctions) throws SQLException {
+
+ validateConditionIsNotEmpty(condition);
+
+ Connection conn = getConnection();
+ PreparedStatement stmt = null;
+ ResultSet rs = null;
+ TimelineMetrics metrics = new TimelineMetrics();
+
+ try {
+ //get latest
+ if(condition.isPointInTime()) {
+ getLatestAggregateMetricRecords(condition, conn, metrics, metricFunctions);
+ } else {
+ stmt = PhoenixTransactSQL.prepareGetAggregateSqlStmt(conn, condition);
+
+ rs = stmt.executeQuery();
+ while (rs.next()) {
+ appendAggregateMetricFromResultSet(metrics, condition, metricFunctions, rs);
+ }
+ }
+ } finally {
+ if (rs != null) {
+ try {
+ rs.close();
+ } catch (SQLException e) {
+ // Ignore
+ }
+ }
+ if (stmt != null) {
+ try {
+ stmt.close();
+ } catch (SQLException e) {
+ // Ignore
+ }
+ }
+ if (conn != null) {
+ try {
+ conn.close();
+ } catch (SQLException sql) {
+ // Ignore
+ }
+ }
+ }
+
+ LOG.debug("Aggregate records size: " + metrics.getMetrics().size());
+ return metrics;
+ }
+
+ private void appendAggregateMetricFromResultSet(TimelineMetrics metrics,
+ Condition condition, Multimap<String, List<Function>> metricFunctions,
+ ResultSet rs) throws SQLException {
+
+ byte[] uuid = rs.getBytes("UUID");
+ String metricName = metadataManagerInstance.getMetricNameFromUuid(uuid);
+ Collection<List<Function>> functionList = findMetricFunctions(metricFunctions, metricName);
+
+ for (List<Function> functions : functionList) {
+ for (Function aggregateFunction : functions) {
+ SingleValuedTimelineMetric metric;
+
+ if (condition.getPrecision() == Precision.MINUTES
+ || condition.getPrecision() == Precision.HOURS
+ || condition.getPrecision() == Precision.DAYS) {
+ metric = getAggregateTimelineMetricFromResultSet(rs, aggregateFunction, false);
+ } else {
+ metric = getAggregateTimelineMetricFromResultSet(rs, aggregateFunction, true);
+ }
+
+ if (condition.isGrouped()) {
+ metrics.addOrMergeTimelineMetric(metric);
+ } else {
+ metrics.getMetrics().add(metric.getTimelineMetric());
+ }
+ }
+ }
+
+ }
+
+ private void getLatestAggregateMetricRecords(Condition condition,
+ Connection conn, TimelineMetrics metrics,
+ Multimap<String, List<Function>> metricFunctions) throws SQLException {
+
+ PreparedStatement stmt = null;
+ SplitByMetricNamesCondition splitCondition =
+ new SplitByMetricNamesCondition(condition);
+
+ for (byte[] uuid: condition.getUuids()) {
+
+ splitCondition.setCurrentUuid(uuid);
+ stmt = PhoenixTransactSQL.prepareGetLatestAggregateMetricSqlStmt(conn, splitCondition);
+ ResultSet rs = null;
+ try {
+ rs = stmt.executeQuery();
+ while (rs.next()) {
+ String metricName = metadataManagerInstance.getMetricNameFromUuid(uuid);
+ Collection<List<Function>> functionList = findMetricFunctions(metricFunctions, metricName);
+ for (List<Function> functions : functionList) {
+ if (functions != null) {
+ for (Function f : functions) {
+ SingleValuedTimelineMetric metric =
+ getAggregateTimelineMetricFromResultSet(rs, f, true);
+
+ if (condition.isGrouped()) {
+ metrics.addOrMergeTimelineMetric(metric);
+ } else {
+ metrics.getMetrics().add(metric.getTimelineMetric());
+ }
+ }
+ } else {
+ SingleValuedTimelineMetric metric =
+ getAggregateTimelineMetricFromResultSet(rs, new Function(), true);
+ metrics.getMetrics().add(metric.getTimelineMetric());
+ }
+ }
+ }
+ } finally {
+ if (rs != null) {
+ try {
+ rs.close();
+ } catch (SQLException e) {
+ // Ignore
+ }
+ }
+ if (stmt != null) {
+ stmt.close();
+ }
+ }
+ }
+ }
+
+ private SingleValuedTimelineMetric getAggregateTimelineMetricFromResultSet(ResultSet rs,
+ Function f, boolean useHostCount) throws SQLException {
+
+ String countColumnName = "METRIC_COUNT";
+ if (useHostCount) {
+ countColumnName = "HOSTS_COUNT";
+ }
+
+ byte[] uuid = rs.getBytes("UUID");
+ TimelineMetric timelineMetric = metadataManagerInstance.getMetricFromUuid(uuid);
+
+ SingleValuedTimelineMetric metric = new SingleValuedTimelineMetric(
+ timelineMetric.getMetricName() + f.getSuffix(),
+ timelineMetric.getAppId(),
+ timelineMetric.getInstanceId(),
+ null,
+ rs.getLong("SERVER_TIME")
+ );
+
+ double value;
+ switch(f.getReadFunction()){
+ case AVG:
+ value = rs.getDouble("METRIC_SUM") / rs.getInt(countColumnName);
+ break;
+ case MIN:
+ value = rs.getDouble("METRIC_MIN");
+ break;
+ case MAX:
+ value = rs.getDouble("METRIC_MAX");
+ break;
+ case SUM:
+ value = rs.getDouble("METRIC_SUM");
+ break;
+ default:
+ value = rs.getDouble("METRIC_SUM") / rs.getInt(countColumnName);
+ break;
+ }
+
+ metric.setSingleTimeseriesValue(rs.getLong("SERVER_TIME"), value);
+
+ return metric;
+ }
+
+ private void validateConditionIsNotEmpty(Condition condition) {
+ if (condition.isEmpty()) {
+ throw new IllegalArgumentException("No filter criteria specified.");
+ }
+ }
+
+ private Collection<List<Function>> findMetricFunctions(Multimap<String, List<Function>> metricFunctions,
+ String metricName) {
+
+ if (metricFunctions == null) {
+ return Collections.singletonList(Collections.singletonList(new Function()));
+ }
+
+ if (metricFunctions.containsKey(metricName)) {
+ return metricFunctions.get(metricName);
+ }
+
+ for (String metricNameEntry : metricFunctions.keySet()) {
+
+ String metricRegEx;
+ //Special case handling for metric name with * and __%.
+ //For example, dfs.NNTopUserOpCounts.windowMs=300000.op=*.user=%.count
+ // or dfs.NNTopUserOpCounts.windowMs=300000.op=__%.user=%.count
+ if (metricNameEntry.contains("*") || metricNameEntry.contains("__%")) {
+ String metricNameWithEscSeq = metricNameEntry.replace("*", "\\*").replace("__%", "..%");
+ metricRegEx = metricNameWithEscSeq.replace("%", ".*");
+ } else {
+ metricRegEx = metricNameEntry.replace("%", ".*");
+ }
+ if (metricName.matches(metricRegEx)) {
+ return metricFunctions.get(metricNameEntry);
+ }
+ }
+
+ return Collections.singletonList(Collections.singletonList(new Function()));
+ }
+
+ public void saveHostAggregateRecords(Map<TimelineMetric, MetricHostAggregate> hostAggregateMap,
+ String phoenixTableName) throws SQLException {
+
+ if (hostAggregateMap == null || hostAggregateMap.isEmpty()) {
+ LOG.debug("Empty aggregate records.");
+ return;
+ }
+
+ Connection conn = getConnection();
+ PreparedStatement stmt = null;
+
+ long start = System.currentTimeMillis();
+ int rowCount = 0;
+
+ try {
+ stmt = conn.prepareStatement(
+ String.format(PhoenixTransactSQL.UPSERT_AGGREGATE_RECORD_SQL, phoenixTableName));
+
+ for (Map.Entry<TimelineMetric, MetricHostAggregate> metricAggregate :
+ hostAggregateMap.entrySet()) {
+
+ TimelineMetric metric = metricAggregate.getKey();
+ MetricHostAggregate hostAggregate = metricAggregate.getValue();
+
+ byte[] uuid = metadataManagerInstance.getUuid(metric);
+ if (uuid == null) {
+ LOG.error("Error computing UUID for metric. Cannot write metric : " + metric.toString());
+ continue;
+ }
+ rowCount++;
+ stmt.clearParameters();
+ stmt.setBytes(1, uuid);
+ stmt.setLong(2, metric.getStartTime());
+ stmt.setDouble(3, hostAggregate.getSum());
+ stmt.setDouble(4, hostAggregate.getMax());
+ stmt.setDouble(5, hostAggregate.getMin());
+ stmt.setDouble(6, hostAggregate.getNumberOfSamples());
+
+ try {
+ stmt.executeUpdate();
+ } catch (SQLException sql) {
+ LOG.error(sql);
+ }
+
+ if (rowCount >= PHOENIX_MAX_MUTATION_STATE_SIZE - 1) {
+ conn.commit();
+ rowCount = 0;
+ }
+
+ }
+
+ conn.commit();
+
+ } finally {
+ if (stmt != null) {
+ try {
+ stmt.close();
+ } catch (SQLException e) {
+ // Ignore
+ }
+ }
+ if (conn != null) {
+ try {
+ conn.close();
+ } catch (SQLException sql) {
+ // Ignore
+ }
+ }
+ }
+
+ long end = System.currentTimeMillis();
+
+ if ((end - start) > 60000l) {
+ LOG.info("Time to save map: " + (end - start) + ", " +
+ "thread = " + Thread.currentThread().getClass());
+ }
+ if (aggregatorSink != null) {
+ try {
+ aggregatorSink.saveHostAggregateRecords(hostAggregateMap,
+ getTablePrecision(phoenixTableName));
+ } catch (Exception e) {
+ LOG.warn(
+ "Error writing host aggregate records metrics to external sink. "
+ + e);
+ }
+ }
+ }
+
+ /**
+ * Save Metric aggregate records.
+ *
+ * @throws SQLException
+ */
+ public void saveClusterAggregateRecords(Map<TimelineClusterMetric, MetricClusterAggregate> records)
+ throws SQLException {
+
+ if (records == null || records.isEmpty()) {
+ LOG.debug("Empty aggregate records.");
+ return;
+ }
+
+ long start = System.currentTimeMillis();
+ String sqlStr = String.format(PhoenixTransactSQL.UPSERT_CLUSTER_AGGREGATE_SQL, PhoenixTransactSQL.METRICS_CLUSTER_AGGREGATE_TABLE_NAME);
+ Connection conn = getConnection();
+ PreparedStatement stmt = null;
+ try {
+ stmt = conn.prepareStatement(sqlStr);
+ int rowCount = 0;
+
+ for (Map.Entry<TimelineClusterMetric, MetricClusterAggregate>
+ aggregateEntry : records.entrySet()) {
+ TimelineClusterMetric clusterMetric = aggregateEntry.getKey();
+ MetricClusterAggregate aggregate = aggregateEntry.getValue();
+
+ if (LOG.isTraceEnabled()) {
+ LOG.trace("clusterMetric = " + clusterMetric + ", " +
+ "aggregate = " + aggregate);
+ }
+
+ rowCount++;
+ byte[] uuid = metadataManagerInstance.getUuid(clusterMetric);
+ if (uuid == null) {
+ LOG.error("Error computing UUID for metric. Cannot write metrics : " + clusterMetric.toString());
+ continue;
+ }
+ stmt.clearParameters();
+ stmt.setBytes(1, uuid);
+ stmt.setLong(2, clusterMetric.getTimestamp());
+ stmt.setDouble(3, aggregate.getSum());
+ stmt.setInt(4, aggregate.getNumberOfHosts());
+ stmt.setDouble(5, aggregate.getMax());
+ stmt.setDouble(6, aggregate.getMin());
+
+ try {
+ stmt.executeUpdate();
+ } catch (SQLException sql) {
+ // we have no way to verify it works!!!
+ LOG.error(sql);
+ }
+
+ if (rowCount >= PHOENIX_MAX_MUTATION_STATE_SIZE - 1) {
+ conn.commit();
+ rowCount = 0;
+ }
+ }
+
+ conn.commit();
+
+ } finally {
+ if (stmt != null) {
+ try {
+ stmt.close();
+ } catch (SQLException e) {
+ // Ignore
+ }
+ }
+ if (conn != null) {
+ try {
+ conn.close();
+ } catch (SQLException sql) {
+ // Ignore
+ }
+ }
+ }
+ long end = System.currentTimeMillis();
+ if ((end - start) > 60000l) {
+ LOG.info("Time to save: " + (end - start) + ", " +
+ "thread = " + Thread.currentThread().getName());
+ }
+ if (aggregatorSink != null) {
+ try {
+ aggregatorSink.saveClusterAggregateRecords(records);
+ } catch (Exception e) {
+ LOG.warn("Error writing cluster aggregate records metrics to external sink. ", e);
+ }
+ }
+ }
+
+
+ /**
+ * Save Metric aggregate records.
+ *
+ * @throws SQLException
+ */
+ public void saveClusterAggregateRecordsSecond(Map<TimelineClusterMetric, MetricHostAggregate> records,
+ String tableName) throws SQLException {
+ if (records == null || records.isEmpty()) {
+ LOG.debug("Empty aggregate records.");
+ return;
+ }
+
+ long start = System.currentTimeMillis();
+
+ Connection conn = getConnection();
+ PreparedStatement stmt = null;
+ try {
+ stmt = conn.prepareStatement(String.format(PhoenixTransactSQL.UPSERT_CLUSTER_AGGREGATE_TIME_SQL, tableName));
+ int rowCount = 0;
+
+ for (Map.Entry<TimelineClusterMetric, MetricHostAggregate> aggregateEntry : records.entrySet()) {
+ TimelineClusterMetric clusterMetric = aggregateEntry.getKey();
+ MetricHostAggregate aggregate = aggregateEntry.getValue();
+
+ if (LOG.isTraceEnabled()) {
+ LOG.trace("clusterMetric = " + clusterMetric + ", " +
+ "aggregate = " + aggregate);
+ }
+
+ byte[] uuid = metadataManagerInstance.getUuid(clusterMetric);
+ if (uuid == null) {
+ LOG.error("Error computing UUID for metric. Cannot write metric : " + clusterMetric.toString());
+ continue;
+ }
+
+ rowCount++;
+ stmt.clearParameters();
+ stmt.setBytes(1, uuid);
+ stmt.setLong(2, clusterMetric.getTimestamp());
+ stmt.setDouble(3, aggregate.getSum());
+ stmt.setLong(4, aggregate.getNumberOfSamples());
+ stmt.setDouble(5, aggregate.getMax());
+ stmt.setDouble(6, aggregate.getMin());
+
+ try {
+ stmt.executeUpdate();
+ } catch (SQLException sql) {
+ // we have no way to verify it works!!!
+ LOG.error(sql);
+ }
+
+ if (rowCount >= PHOENIX_MAX_MUTATION_STATE_SIZE - 1) {
+ conn.commit();
+ rowCount = 0;
+ }
+ }
+
+ conn.commit();
+
+ } finally {
+ if (stmt != null) {
+ try {
+ stmt.close();
+ } catch (SQLException e) {
+ // Ignore
+ }
+ }
+ if (conn != null) {
+ try {
+ conn.close();
+ } catch (SQLException sql) {
+ // Ignore
+ }
+ }
+ }
+ long end = System.currentTimeMillis();
+ if ((end - start) > 60000l) {
+ LOG.info("Time to save: " + (end - start) + ", " +
+ "thread = " + Thread.currentThread().getName());
+ }
+ if (aggregatorSink != null) {
+ try {
+ aggregatorSink.saveClusterTimeAggregateRecords(records,
+ getTablePrecision(tableName));
+ } catch (Exception e) {
+ LOG.warn(
+ "Error writing cluster time aggregate records metrics to external sink. "
+ + e);
+ }
+ }
+ }
+
+ /**
+ * Get precision for a table
+ * @param tableName
+ * @return precision
+ */
+ private Precision getTablePrecision(String tableName) {
+ Precision tablePrecision = null;
+ switch (tableName) {
+ case PhoenixTransactSQL.METRICS_RECORD_TABLE_NAME:
+ tablePrecision = Precision.SECONDS;
+ break;
+ case PhoenixTransactSQL.METRICS_AGGREGATE_MINUTE_TABLE_NAME:
+ case PhoenixTransactSQL.METRICS_CLUSTER_AGGREGATE_MINUTE_TABLE_NAME:
+ tablePrecision = Precision.MINUTES;
+ break;
+ case PhoenixTransactSQL.METRICS_AGGREGATE_HOURLY_TABLE_NAME:
+ case PhoenixTransactSQL.METRICS_CLUSTER_AGGREGATE_HOURLY_TABLE_NAME:
+ tablePrecision = Precision.HOURS;
+ break;
+ case PhoenixTransactSQL.METRICS_AGGREGATE_DAILY_TABLE_NAME:
+ case PhoenixTransactSQL.METRICS_CLUSTER_AGGREGATE_DAILY_TABLE_NAME:
+ tablePrecision = Precision.DAYS;
+ break;
+ }
+ return tablePrecision;
+ }
+
+ /**
+ * Provide skip block cache hint for aggregator queries.
+ */
+ public boolean isSkipBlockCacheForAggregatorsEnabled() {
+ return skipBlockCacheForAggregatorsEnabled;
+ }
+
+ /**
+ * One time save of metadata when discovering topology during aggregation.
+ * @throws SQLException
+ */
+ public void saveHostAppsMetadata(Map<String, TimelineMetricHostMetadata> hostMetadata) throws SQLException {
+ Connection conn = getConnection();
+ PreparedStatement stmt = null;
+ try {
+ stmt = conn.prepareStatement(PhoenixTransactSQL.UPSERT_HOSTED_APPS_METADATA_SQL);
+ int rowCount = 0;
+
+ for (Map.Entry<String, TimelineMetricHostMetadata> hostedAppsEntry : hostMetadata.entrySet()) {
+ TimelineMetricHostMetadata timelineMetricHostMetadata = hostedAppsEntry.getValue();
+ if (LOG.isTraceEnabled()) {
+ LOG.trace("HostedAppsMetadata: " + hostedAppsEntry);
+ }
+
+ stmt.clearParameters();
+ stmt.setString(1, hostedAppsEntry.getKey());
+ stmt.setBytes(2, timelineMetricHostMetadata.getUuid());
+ stmt.setString(3, StringUtils.join(timelineMetricHostMetadata.getHostedApps().keySet(), ","));
+ try {
+ stmt.executeUpdate();
+ rowCount++;
+ } catch (SQLException sql) {
+ LOG.error("Error saving hosted apps metadata.", sql);
+ }
+ }
+
+ conn.commit();
+ LOG.info("Saved " + rowCount + " hosted apps metadata records.");
+
+ } finally {
+ if (stmt != null) {
+ try {
+ stmt.close();
+ } catch (SQLException e) {
+ // Ignore
+ }
+ }
+ if (conn != null) {
+ try {
+ conn.close();
+ } catch (SQLException sql) {
+ // Ignore
+ }
+ }
+ }
+ }
+
+ public void saveInstanceHostsMetadata(Map<String, Set<String>> instanceHostsMap) throws SQLException {
+ Connection conn = getConnection();
+ PreparedStatement stmt = null;
+ try {
+ stmt = conn.prepareStatement(PhoenixTransactSQL.UPSERT_INSTANCE_HOST_METADATA_SQL);
+ int rowCount = 0;
+
+ for (Map.Entry<String, Set<String>> hostInstancesEntry : instanceHostsMap.entrySet()) {
+ if (LOG.isTraceEnabled()) {
+ LOG.trace("Host Instances Entry: " + hostInstancesEntry);
+ }
+
+ String instanceId = hostInstancesEntry.getKey();
+
+ for(String hostname : hostInstancesEntry.getValue()) {
+ stmt.clearParameters();
+ stmt.setString(1, instanceId);
+ stmt.setString(2, hostname);
+ try {
+ stmt.executeUpdate();
+ rowCount++;
+ } catch (SQLException sql) {
+ LOG.error("Error saving host instances metadata.", sql);
+ }
+ }
+
+ }
+
+ conn.commit();
+ LOG.info("Saved " + rowCount + " host instances metadata records.");
+
+ } finally {
+ if (stmt != null) {
+ try {
+ stmt.close();
+ } catch (SQLException e) {
+ // Ignore
+ }
+ }
+ if (conn != null) {
+ try {
+ conn.close();
+ } catch (SQLException sql) {
+ // Ignore
+ }
+ }
+ }
+ }
+
+ /**
+ * Save metdata on updates.
+ * @param metricMetadata @Collection<@TimelineMetricMetadata>
+ * @throws SQLException
+ */
+ public void saveMetricMetadata(Collection<TimelineMetricMetadata> metricMetadata) throws SQLException {
+ if (metricMetadata.isEmpty()) {
+ LOG.info("No metadata records to save.");
+ return;
+ }
+
+ Connection conn = getConnection();
+ PreparedStatement stmt = null;
+
+ try {
+ stmt = conn.prepareStatement(PhoenixTransactSQL.UPSERT_METADATA_SQL);
+ int rowCount = 0;
+
+ for (TimelineMetricMetadata metadata : metricMetadata) {
+ if (LOG.isTraceEnabled()) {
+ LOG.trace("TimelineMetricMetadata: metricName = " + metadata.getMetricName()
+ + ", appId = " + metadata.getAppId()
+ + ", seriesStartTime = " + metadata.getSeriesStartTime()
+ );
+ }
+ try {
+ stmt.clearParameters();
+ stmt.setString(1, metadata.getMetricName());
+ stmt.setString(2, metadata.getAppId());
+ stmt.setString(3, metadata.getInstanceId());
+ stmt.setBytes(4, metadata.getUuid());
+ stmt.setString(5, metadata.getUnits());
+ stmt.setString(6, metadata.getType());
+ stmt.setLong(7, metadata.getSeriesStartTime());
+ stmt.setBoolean(8, metadata.isSupportsAggregates());
+ stmt.setBoolean(9, metadata.isWhitelisted());
+ } catch (Exception e) {
+ LOG.error("Exception in saving metric metadata entry. ");
+ continue;
+ }
+
+ try {
+ stmt.executeUpdate();
+ rowCount++;
+ } catch (SQLException sql) {
+ LOG.error("Error saving metadata.", sql);
+ }
+ }
+
+ conn.commit();
+ LOG.info("Saved " + rowCount + " metadata records.");
+
+ } finally {
+ if (stmt != null) {
+ try {
+ stmt.close();
+ } catch (SQLException e) {
+ // Ignore
+ }
+ }
+ if (conn != null) {
+ try {
+ conn.close();
+ } catch (SQLException sql) {
+ // Ignore
+ }
+ }
+ }
+ }
+
+ public Map<String, TimelineMetricHostMetadata> getHostedAppsMetadata() throws SQLException {
+ Map<String, TimelineMetricHostMetadata> hostedAppMap = new HashMap<>();
+ Connection conn = getConnection();
+ PreparedStatement stmt = null;
+ ResultSet rs = null;
+
+ try {
+ stmt = conn.prepareStatement(PhoenixTransactSQL.GET_HOSTED_APPS_METADATA_SQL);
+ rs = stmt.executeQuery();
+
+ while (rs.next()) {
+ TimelineMetricHostMetadata hostMetadata = new TimelineMetricHostMetadata(new HashSet<>(Arrays.asList(StringUtils.split(rs.getString("APP_IDS"), ","))));
+ hostMetadata.setUuid(rs.getBytes("UUID"));
+ hostedAppMap.put(rs.getString("HOSTNAME"), hostMetadata);
+ }
+
+ } finally {
+ if (rs != null) {
+ try {
+ rs.close();
+ } catch (SQLException e) {
+ // Ignore
+ }
+ }
+ if (stmt != null) {
+ try {
+ stmt.close();
+ } catch (SQLException e) {
+ // Ignore
+ }
+ }
+ if (conn != null) {
+ try {
+ conn.close();
+ } catch (SQLException sql) {
+ // Ignore
+ }
+ }
+ }
+
+ return hostedAppMap;
+ }
+
+ public Map<String, Set<String>> getInstanceHostsMetdata() throws SQLException {
+ Map<String, Set<String>> instanceHostsMap = new HashMap<>();
+ Connection conn = getConnection();
+ PreparedStatement stmt = null;
+ ResultSet rs = null;
+
+ try {
+ stmt = conn.prepareStatement(PhoenixTransactSQL.GET_INSTANCE_HOST_METADATA_SQL);
+ rs = stmt.executeQuery();
+
+ while (rs.next()) {
+ String instanceId = rs.getString("INSTANCE_ID");
+ String hostname = rs.getString("HOSTNAME");
+
+ if (!instanceHostsMap.containsKey(instanceId)) {
+ instanceHostsMap.put(instanceId, new HashSet<String>());
+ }
+ instanceHostsMap.get(instanceId).add(hostname);
+ }
+
+ } finally {
+ if (rs != null) {
+ try {
+ rs.close();
+ } catch (SQLException e) {
+ // Ignore
+ }
+ }
+ if (stmt != null) {
+ try {
+ stmt.close();
+ } catch (SQLException e) {
+ // Ignore
+ }
+ }
+ if (conn != null) {
+ try {
+ conn.close();
+ } catch (SQLException sql) {
+ // Ignore
+ }
+ }
+ }
+
+ return instanceHostsMap;
+ }
+
+ // No filter criteria support for now.
+ public Map<TimelineMetricMetadataKey, TimelineMetricMetadata> getTimelineMetricMetadata() throws SQLException {
+ Map<TimelineMetricMetadataKey, TimelineMetricMetadata> metadataMap = new HashMap<>();
+ Connection conn = getConnection();
+ PreparedStatement stmt = null;
+ ResultSet rs = null;
+
+ try {
+ stmt = conn.prepareStatement(PhoenixTransactSQL.GET_METRIC_METADATA_SQL);
+ rs = stmt.executeQuery();
+
+ while (rs.next()) {
+ String metricName = rs.getString("METRIC_NAME");
+ String appId = rs.getString("APP_ID");
+ String instanceId = rs.getString("INSTANCE_ID");
+ TimelineMetricMetadata metadata = new TimelineMetricMetadata(
+ metricName,
+ appId,
+ instanceId,
+ rs.getString("UNITS"),
+ rs.getString("TYPE"),
+ rs.getLong("START_TIME"),
+ rs.getBoolean("SUPPORTS_AGGREGATION"),
+ rs.getBoolean("IS_WHITELISTED")
+ );
+
+ TimelineMetricMetadataKey key = new TimelineMetricMetadataKey(metricName, appId, instanceId);
+ metadata.setIsPersisted(true); // Always true on retrieval
+ metadata.setUuid(rs.getBytes("UUID"));
+ metadataMap.put(key, metadata);
+ }
+
+ } finally {
+ if (rs != null) {
+ try {
+ rs.close();
+ } catch (SQLException e) {
+ // Ignore
+ }
+ }
+ if (stmt != null) {
+ try {
+ stmt.close();
+ } catch (SQLException e) {
+ // Ignore
+ }
+ }
+ if (conn != null) {
+ try {
+ conn.close();
+ } catch (SQLException sql) {
+ // Ignore
+ }
+ }
+ }
+
+ return metadataMap;
+ }
+
+ public void setMetadataInstance(TimelineMetricMetadataManager metadataManager) {
+ this.metadataManagerInstance = metadataManager;
+ TIMELINE_METRIC_READ_HELPER = new TimelineMetricReadHelper(this.metadataManagerInstance);
+ }
+}
[06/28] ambari git commit: AMBARI-22740 : Fix integration test for
HBase in branch-3.0-ams due to UUID changes. (avijayan)
Posted by av...@apache.org.
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/timeline/aggregators/ITMetricAggregator.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/timeline/aggregators/ITMetricAggregator.java b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/timeline/aggregators/ITMetricAggregator.java
new file mode 100644
index 0000000..8517105
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/timeline/aggregators/ITMetricAggregator.java
@@ -0,0 +1,355 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.metrics.core.timeline.aggregators;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.metrics2.sink.timeline.MetricHostAggregate;
+import org.apache.hadoop.metrics2.sink.timeline.TimelineMetric;
+import org.apache.hadoop.metrics2.sink.timeline.TimelineMetrics;
+import org.apache.ambari.metrics.core.timeline.AbstractMiniHBaseClusterTest;
+import org.apache.ambari.metrics.core.timeline.MetricTestHelper;
+import org.apache.ambari.metrics.core.timeline.query.Condition;
+import org.apache.ambari.metrics.core.timeline.query.DefaultCondition;
+import org.apache.ambari.metrics.core.timeline.query.PhoenixTransactSQL;
+import org.junit.Test;
+
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.TreeMap;
+
+import static junit.framework.Assert.assertEquals;
+import static junit.framework.Assert.assertTrue;
+import static junit.framework.Assert.fail;
+import static org.apache.ambari.metrics.core.timeline.MetricTestHelper.createEmptyTimelineMetric;
+import static org.apache.ambari.metrics.core.timeline.query.PhoenixTransactSQL.GET_METRIC_AGGREGATE_ONLY_SQL;
+import static org.apache.ambari.metrics.core.timeline.query.PhoenixTransactSQL.METRICS_AGGREGATE_DAILY_TABLE_NAME;
+import static org.apache.ambari.metrics.core.timeline.query.PhoenixTransactSQL.METRICS_AGGREGATE_HOURLY_TABLE_NAME;
+import static org.apache.ambari.metrics.core.timeline.query.PhoenixTransactSQL.METRICS_AGGREGATE_MINUTE_TABLE_NAME;
+import static org.assertj.core.api.Assertions.assertThat;
+
+public class ITMetricAggregator extends AbstractMiniHBaseClusterTest {
+
+ @Test
+ public void testShouldInsertMetrics() throws Exception {
+ // GIVEN
+
+ // WHEN
+ long startTime = System.currentTimeMillis();
+ TimelineMetrics metricsSent = prepareTimelineMetrics(startTime, "local");
+ hdb.insertMetricRecords(metricsSent, true);
+
+ Condition queryCondition = new DefaultCondition(null,
+ Collections.singletonList("local"), null, null, startTime,
+ startTime + (15 * 60 * 1000), null, null, false);
+ TimelineMetrics recordRead = hdb.getMetricRecords(queryCondition, null);
+
+ // THEN
+ assertThat(recordRead.getMetrics()).hasSize(2)
+ .extracting("metricName")
+ .containsOnly("mem_free", "disk_free");
+
+ assertThat(metricsSent.getMetrics())
+ .usingElementComparator(TIME_IGNORING_COMPARATOR)
+ .containsAll(recordRead.getMetrics());
+ }
+
+ private Configuration getConfigurationForTest(boolean useGroupByAggregators) {
+ Configuration configuration = new Configuration();
+ configuration.set("timeline.metrics.service.use.groupBy.aggregators", String.valueOf(useGroupByAggregators));
+ return configuration;
+ }
+
+ @Test
+ public void testShouldAggregateMinuteProperly() throws Exception {
+ // GIVEN
+ TimelineMetricAggregator aggregatorMinute =
+ TimelineMetricAggregatorFactory.createTimelineMetricAggregatorMinute(hdb,
+ getConfigurationForTest(false), metadataManager, null);
+ TimelineMetricReadHelper readHelper = new TimelineMetricReadHelper(metadataManager, false);
+
+ long startTime = System.currentTimeMillis();
+ long ctime = startTime;
+ long minute = 60 * 1000;
+ hdb.insertMetricRecords(prepareTimelineMetrics(startTime, "local"), true);
+ hdb.insertMetricRecords(prepareTimelineMetrics(ctime += minute, "local"), true);
+ hdb.insertMetricRecords(prepareTimelineMetrics(ctime += minute, "local"), true);
+ hdb.insertMetricRecords(prepareTimelineMetrics(ctime += minute, "local"), true);
+ hdb.insertMetricRecords(prepareTimelineMetrics(ctime += minute, "local"), true);
+
+ // WHEN
+ long endTime = startTime + 1000 * 60 * 4;
+ boolean success = aggregatorMinute.doWork(startTime, endTime + 1);
+
+ //THEN
+ Condition condition = new DefaultCondition(null, null, null, null, startTime,
+ endTime + 2, null, null, true);
+ condition.setStatement(String.format(GET_METRIC_AGGREGATE_ONLY_SQL,
+ METRICS_AGGREGATE_MINUTE_TABLE_NAME));
+
+ PreparedStatement pstmt = PhoenixTransactSQL.prepareGetMetricsSqlStmt(conn, condition);
+ ResultSet rs = pstmt.executeQuery();
+ MetricHostAggregate expectedAggregate =
+ MetricTestHelper.createMetricHostAggregate(2.0, 0.0, 20, 15.0);
+
+ int count = 0;
+ while (rs.next()) {
+ TimelineMetric currentMetric =
+ readHelper.getTimelineMetricKeyFromResultSet(rs);
+ MetricHostAggregate currentHostAggregate =
+ readHelper.getMetricHostAggregateFromResultSet(rs);
+
+ if ("disk_free".equals(currentMetric.getMetricName())) {
+ assertEquals(2.0, currentHostAggregate.getMax());
+ assertEquals(0.0, currentHostAggregate.getMin());
+ assertEquals(20, currentHostAggregate.getNumberOfSamples());
+ assertEquals(15.0, currentHostAggregate.getSum());
+ assertEquals(15.0 / 20, currentHostAggregate.calculateAverage());
+ count++;
+ } else if ("mem_free".equals(currentMetric.getMetricName())) {
+ assertEquals(2.0, currentHostAggregate.getMax());
+ assertEquals(0.0, currentHostAggregate.getMin());
+ assertEquals(20, currentHostAggregate.getNumberOfSamples());
+ assertEquals(15.0, currentHostAggregate.getSum());
+ assertEquals(15.0 / 20, currentHostAggregate.calculateAverage());
+ count++;
+ } else {
+ fail("Unexpected entry");
+ }
+ }
+ assertEquals("Two aggregated entries expected", 2, count);
+ }
+
+ @Test
+ public void testShouldAggregateHourProperly() throws Exception {
+ // GIVEN
+ TimelineMetricAggregator aggregator =
+ TimelineMetricAggregatorFactory.createTimelineMetricAggregatorHourly(hdb,
+ getConfigurationForTest(false), metadataManager, null);
+ TimelineMetricReadHelper readHelper = new TimelineMetricReadHelper(metadataManager, false);
+ long startTime = System.currentTimeMillis();
+
+ MetricHostAggregate expectedAggregate =
+ MetricTestHelper.createMetricHostAggregate(2.0, 0.0, 20, 15.0);
+ Map<TimelineMetric, MetricHostAggregate>
+ aggMap = new HashMap<TimelineMetric,
+ MetricHostAggregate>();
+
+ int min_5 = 5 * 60 * 1000;
+ long ctime = startTime - min_5;
+ aggMap.put(createEmptyTimelineMetric(ctime += min_5), expectedAggregate);
+ aggMap.put(createEmptyTimelineMetric(ctime += min_5), expectedAggregate);
+ aggMap.put(createEmptyTimelineMetric(ctime += min_5), expectedAggregate);
+ aggMap.put(createEmptyTimelineMetric(ctime += min_5), expectedAggregate);
+ aggMap.put(createEmptyTimelineMetric(ctime += min_5), expectedAggregate);
+ aggMap.put(createEmptyTimelineMetric(ctime += min_5), expectedAggregate);
+ aggMap.put(createEmptyTimelineMetric(ctime += min_5), expectedAggregate);
+ aggMap.put(createEmptyTimelineMetric(ctime += min_5), expectedAggregate);
+ aggMap.put(createEmptyTimelineMetric(ctime += min_5), expectedAggregate);
+ aggMap.put(createEmptyTimelineMetric(ctime += min_5), expectedAggregate);
+ aggMap.put(createEmptyTimelineMetric(ctime += min_5), expectedAggregate);
+ aggMap.put(createEmptyTimelineMetric(ctime += min_5), expectedAggregate);
+
+ hdb.saveHostAggregateRecords(aggMap, METRICS_AGGREGATE_MINUTE_TABLE_NAME);
+
+ //WHEN
+ long endTime = ctime + min_5;
+ boolean success = aggregator.doWork(startTime, endTime);
+ assertTrue(success);
+
+ //THEN
+ Condition condition = new DefaultCondition(null, null, null, null, startTime,
+ endTime + 1, null, null, true);
+ condition.setStatement(String.format(GET_METRIC_AGGREGATE_ONLY_SQL, METRICS_AGGREGATE_HOURLY_TABLE_NAME));
+
+ PreparedStatement pstmt = PhoenixTransactSQL.prepareGetMetricsSqlStmt(conn, condition);
+ ResultSet rs = pstmt.executeQuery();
+
+ while (rs.next()) {
+ TimelineMetric currentMetric =
+ readHelper.getTimelineMetricKeyFromResultSet(rs);
+ MetricHostAggregate currentHostAggregate =
+ readHelper.getMetricHostAggregateFromResultSet(rs);
+
+ if ("disk_used".equals(currentMetric.getMetricName())) {
+ assertEquals(2.0, currentHostAggregate.getMax());
+ assertEquals(0.0, currentHostAggregate.getMin());
+ assertEquals(12 * 20, currentHostAggregate.getNumberOfSamples());
+ assertEquals(12 * 15.0, currentHostAggregate.getSum());
+ assertEquals(15.0 / 20, currentHostAggregate.calculateAverage());
+ }
+ }
+ }
+
+ @Test
+ public void testMetricAggregateDaily() throws Exception {
+ // GIVEN
+ TimelineMetricAggregator aggregator =
+ TimelineMetricAggregatorFactory.createTimelineMetricAggregatorDaily(hdb,
+ getConfigurationForTest(false), metadataManager, null);
+ TimelineMetricReadHelper readHelper = new TimelineMetricReadHelper(metadataManager, false);
+ long startTime = System.currentTimeMillis();
+
+ MetricHostAggregate expectedAggregate =
+ MetricTestHelper.createMetricHostAggregate(2.0, 0.0, 20, 15.0);
+ Map<TimelineMetric, MetricHostAggregate>
+ aggMap = new HashMap<TimelineMetric, MetricHostAggregate>();
+
+ int min_5 = 5 * 60 * 1000;
+ long ctime = startTime - min_5;
+ aggMap.put(createEmptyTimelineMetric(ctime += min_5), expectedAggregate);
+ aggMap.put(createEmptyTimelineMetric(ctime += min_5), expectedAggregate);
+ aggMap.put(createEmptyTimelineMetric(ctime += min_5), expectedAggregate);
+ aggMap.put(createEmptyTimelineMetric(ctime += min_5), expectedAggregate);
+ aggMap.put(createEmptyTimelineMetric(ctime += min_5), expectedAggregate);
+ aggMap.put(createEmptyTimelineMetric(ctime += min_5), expectedAggregate);
+ aggMap.put(createEmptyTimelineMetric(ctime += min_5), expectedAggregate);
+ aggMap.put(createEmptyTimelineMetric(ctime += min_5), expectedAggregate);
+ aggMap.put(createEmptyTimelineMetric(ctime += min_5), expectedAggregate);
+ aggMap.put(createEmptyTimelineMetric(ctime += min_5), expectedAggregate);
+ aggMap.put(createEmptyTimelineMetric(ctime += min_5), expectedAggregate);
+ aggMap.put(createEmptyTimelineMetric(ctime += min_5), expectedAggregate);
+
+ hdb.saveHostAggregateRecords(aggMap, METRICS_AGGREGATE_HOURLY_TABLE_NAME);
+
+ //WHEN
+ long endTime = ctime + min_5;
+ boolean success = aggregator.doWork(startTime, endTime);
+ assertTrue(success);
+
+ //THEN
+ Condition condition = new DefaultCondition(null, null, null, null, startTime,
+ endTime + 1, null, null, true);
+ condition.setStatement(String.format(GET_METRIC_AGGREGATE_ONLY_SQL, METRICS_AGGREGATE_DAILY_TABLE_NAME));
+
+ PreparedStatement pstmt = PhoenixTransactSQL.prepareGetMetricsSqlStmt(conn, condition);
+ ResultSet rs = pstmt.executeQuery();
+
+ while (rs.next()) {
+ TimelineMetric currentMetric =
+ readHelper.getTimelineMetricKeyFromResultSet(rs);
+ MetricHostAggregate currentHostAggregate =
+ readHelper.getMetricHostAggregateFromResultSet(rs);
+
+ if ("disk_used".equals(currentMetric.getMetricName())) {
+ assertEquals(2.0, currentHostAggregate.getMax());
+ assertEquals(0.0, currentHostAggregate.getMin());
+ assertEquals(12 * 20, currentHostAggregate.getNumberOfSamples());
+ assertEquals(12 * 15.0, currentHostAggregate.getSum());
+ assertEquals(15.0 / 20, currentHostAggregate.calculateAverage());
+ }
+ }
+ }
+
+ @Test
+ public void testAggregationUsingGroupByQuery() throws Exception {
+ // GIVEN
+ TimelineMetricAggregator aggregatorMinute =
+ TimelineMetricAggregatorFactory.createTimelineMetricAggregatorMinute(hdb,
+ getConfigurationForTest(true), metadataManager, null);
+ TimelineMetricReadHelper readHelper = new TimelineMetricReadHelper(metadataManager, false);
+
+ long startTime = System.currentTimeMillis();
+ long ctime = startTime;
+ long minute = 60 * 1000;
+ hdb.insertMetricRecords(prepareTimelineMetrics(startTime, "local"), true);
+ hdb.insertMetricRecords(prepareTimelineMetrics(ctime += minute, "local"), true);
+ hdb.insertMetricRecords(prepareTimelineMetrics(ctime += minute, "local"), true);
+ hdb.insertMetricRecords(prepareTimelineMetrics(ctime += minute, "local"), true);
+ hdb.insertMetricRecords(prepareTimelineMetrics(ctime += minute, "local"), true);
+
+ long endTime = startTime + 1000 * 60 * 4;
+ boolean success = aggregatorMinute.doWork(startTime - 1, endTime);
+ assertTrue(success);
+
+ Condition condition = new DefaultCondition(null, null, null, null, startTime,
+ endTime + 1, null, null, true);
+ condition.setStatement(String.format(GET_METRIC_AGGREGATE_ONLY_SQL, METRICS_AGGREGATE_MINUTE_TABLE_NAME));
+
+ PreparedStatement pstmt = PhoenixTransactSQL.prepareGetMetricsSqlStmt(conn, condition);
+ ResultSet rs = pstmt.executeQuery();
+ MetricHostAggregate expectedAggregate =
+ MetricTestHelper.createMetricHostAggregate(2.0, 0.0, 20, 15.0);
+
+ int count = 0;
+ while (rs.next()) {
+ TimelineMetric currentMetric =
+ readHelper.getTimelineMetricKeyFromResultSet(rs);
+ MetricHostAggregate currentHostAggregate =
+ readHelper.getMetricHostAggregateFromResultSet(rs);
+
+ if ("disk_free".equals(currentMetric.getMetricName())) {
+ assertEquals(2.0, currentHostAggregate.getMax());
+ assertEquals(0.0, currentHostAggregate.getMin());
+ assertEquals(20, currentHostAggregate.getNumberOfSamples());
+ assertEquals(15.0, currentHostAggregate.getSum());
+ assertEquals(15.0 / 20, currentHostAggregate.calculateAverage());
+ count++;
+ } else if ("mem_free".equals(currentMetric.getMetricName())) {
+ assertEquals(2.0, currentHostAggregate.getMax());
+ assertEquals(0.0, currentHostAggregate.getMin());
+ assertEquals(20, currentHostAggregate.getNumberOfSamples());
+ assertEquals(15.0, currentHostAggregate.getSum());
+ assertEquals(15.0 / 20, currentHostAggregate.calculateAverage());
+ count++;
+ } else {
+ fail("Unexpected entry");
+ }
+ }
+ assertEquals("Two aggregated entries expected", 2, count);
+ }
+
+ private final static Comparator<TimelineMetric> TIME_IGNORING_COMPARATOR =
+ new Comparator<TimelineMetric>() {
+ @Override
+ public int compare(TimelineMetric o1, TimelineMetric o2) {
+ return o1.equalsExceptTime(o2) ? 0 : 1;
+ }
+ };
+
+ private TimelineMetrics prepareTimelineMetrics(long startTime, String host) {
+ TimelineMetrics metrics = new TimelineMetrics();
+ metrics.setMetrics(Arrays.asList(
+ createMetric(startTime, "disk_free", host),
+ createMetric(startTime, "mem_free", host)));
+
+ return metrics;
+ }
+
+ private TimelineMetric createMetric(long startTime, String metricName, String host) {
+ TimelineMetric m = new TimelineMetric();
+ m.setAppId("host");
+ m.setHostName(host);
+ m.setMetricName(metricName);
+ m.setStartTime(startTime);
+ TreeMap<Long, Double> vals = new TreeMap<Long, Double>();
+ vals.put(startTime + 15000l, 0.0);
+ vals.put(startTime + 30000l, 0.0);
+ vals.put(startTime + 45000l, 1.0);
+ vals.put(startTime + 60000l, 2.0);
+
+ m.setMetricValues(vals);
+
+ return m;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/timeline/aggregators/TimelineMetricClusterAggregatorSecondTest.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/timeline/aggregators/TimelineMetricClusterAggregatorSecondTest.java b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/timeline/aggregators/TimelineMetricClusterAggregatorSecondTest.java
new file mode 100644
index 0000000..c73ac36
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/timeline/aggregators/TimelineMetricClusterAggregatorSecondTest.java
@@ -0,0 +1,404 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.metrics.core.timeline.aggregators;
+
+
+import static org.apache.ambari.metrics.core.timeline.aggregators.AggregatorUtils.getRoundedAggregateTimeMillis;
+import static org.apache.ambari.metrics.core.timeline.aggregators.AggregatorUtils.getRoundedCheckPointTimeMillis;
+import static org.apache.ambari.metrics.core.timeline.aggregators.AggregatorUtils.getTimeSlices;
+import static org.apache.ambari.metrics.core.timeline.aggregators.AggregatorUtils.sliceFromTimelineMetric;
+import static org.apache.ambari.metrics.core.timeline.availability.AggregationTaskRunner.AGGREGATOR_NAME.METRIC_AGGREGATE_SECOND;
+import static org.easymock.EasyMock.anyObject;
+import static org.easymock.EasyMock.createNiceMock;
+import static org.easymock.EasyMock.expect;
+import static org.easymock.EasyMock.replay;
+
+import java.sql.ResultSet;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.TreeMap;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.metrics2.sink.timeline.MetricClusterAggregate;
+import org.apache.hadoop.metrics2.sink.timeline.TimelineMetric;
+import org.apache.ambari.metrics.core.timeline.discovery.TimelineMetricMetadataKey;
+import org.apache.ambari.metrics.core.timeline.discovery.TimelineMetricMetadataManager;
+import org.codehaus.jackson.map.ObjectMapper;
+import org.junit.Test;
+
+ import junit.framework.Assert;
+
+public class TimelineMetricClusterAggregatorSecondTest {
+
+ @Test
+ public void testClusterSecondAggregatorWithInterpolation() {
+
+ long aggregatorInterval = 120000l;
+ long sliceInterval = 30000l;
+ long metricInterval = 10000l;
+
+ TimelineMetricMetadataManager metricMetadataManagerMock = createNiceMock(TimelineMetricMetadataManager.class);
+ expect(metricMetadataManagerMock.getUuid(anyObject(TimelineClusterMetric.class))).andReturn(new byte[16]).once();
+ replay(metricMetadataManagerMock);
+
+ long roundedEndTime = getRoundedAggregateTimeMillis(aggregatorInterval);
+ long roundedStartTime = roundedEndTime - aggregatorInterval;
+ List<Long[]> timeSlices = getTimeSlices(roundedStartTime ,
+ roundedEndTime, sliceInterval);
+
+ TreeMap<Long, Double> metricValues = new TreeMap<>();
+
+ long startTime = roundedEndTime - aggregatorInterval;
+
+ for (int i=1; startTime < roundedEndTime; i++) {
+ startTime += metricInterval;
+ if (i%6 == 1 || i%6 == 2) {
+ metricValues.put(startTime, (double)i);
+ }
+ }
+
+ TimelineMetric counterMetric = new TimelineMetric();
+ counterMetric.setMetricName("TestMetric");
+ counterMetric.setHostName("TestHost");
+ counterMetric.setAppId("TestAppId");
+ counterMetric.setMetricValues(metricValues);
+ counterMetric.setType("COUNTER");
+
+ Map<TimelineClusterMetric, Double> timelineClusterMetricMap = sliceFromTimelineMetric(counterMetric, timeSlices, true);
+
+ TimelineClusterMetric timelineClusterMetric = new TimelineClusterMetric(counterMetric.getMetricName(), counterMetric.getAppId(),
+ counterMetric.getInstanceId(), 0l);
+
+ timelineClusterMetric.setTimestamp(roundedStartTime + 2*sliceInterval);
+ Assert.assertTrue(timelineClusterMetricMap.containsKey(timelineClusterMetric));
+ Assert.assertEquals(timelineClusterMetricMap.get(timelineClusterMetric), 6.0);
+
+ timelineClusterMetric.setTimestamp(roundedStartTime + 4*sliceInterval);
+ Assert.assertTrue(timelineClusterMetricMap.containsKey(timelineClusterMetric));
+ Assert.assertEquals(timelineClusterMetricMap.get(timelineClusterMetric), 12.0);
+
+ TimelineMetric metric = new TimelineMetric();
+ metric.setMetricName("TestMetric");
+ metric.setHostName("TestHost");
+ metric.setAppId("TestAppId");
+ metric.setMetricValues(metricValues);
+
+ timelineClusterMetricMap = sliceFromTimelineMetric(metric, timeSlices, true);
+
+ timelineClusterMetric = new TimelineClusterMetric(metric.getMetricName(), metric.getAppId(),
+ metric.getInstanceId(), 0l);
+
+ timelineClusterMetric.setTimestamp(roundedStartTime + 2*sliceInterval);
+ Assert.assertTrue(timelineClusterMetricMap.containsKey(timelineClusterMetric));
+ Assert.assertEquals(timelineClusterMetricMap.get(timelineClusterMetric), 4.5);
+
+ timelineClusterMetric.setTimestamp(roundedStartTime + 4*sliceInterval);
+ Assert.assertTrue(timelineClusterMetricMap.containsKey(timelineClusterMetric));
+ Assert.assertEquals(timelineClusterMetricMap.get(timelineClusterMetric), 7.5);
+ }
+
+ @Test
+ public void testShouldAggregateProperly() {
+
+ long aggregatorInterval = 120000l;
+ long sliceInterval = 30000l;
+
+ Configuration configuration = new Configuration();
+ TimelineMetricMetadataManager metricMetadataManagerMock = createNiceMock(TimelineMetricMetadataManager.class);
+
+ expect(metricMetadataManagerMock.getMetadataCacheValue((TimelineMetricMetadataKey) anyObject()))
+ .andReturn(null).anyTimes();
+ replay(metricMetadataManagerMock);
+
+ TimelineMetricClusterAggregatorSecond secondAggregator = new TimelineMetricClusterAggregatorSecond(
+ METRIC_AGGREGATE_SECOND, metricMetadataManagerMock, null, configuration, null,
+ aggregatorInterval, 2, "false", "", "", aggregatorInterval, sliceInterval, null
+ );
+
+ long startTime = getRoundedCheckPointTimeMillis(System.currentTimeMillis(),aggregatorInterval);
+ List<Long[]> timeslices = getTimeSlices(startTime, startTime + aggregatorInterval, sliceInterval);
+
+ Map<TimelineClusterMetric, MetricClusterAggregate> aggregateClusterMetrics = new HashMap<>();
+ long seconds = 1000;
+ TimelineMetric timelineMetric = new TimelineMetric();
+ timelineMetric.setMetricName("m1");
+ timelineMetric.setHostName("h1");
+ timelineMetric.setAppId("a1");
+ timelineMetric.setType("GUAGE");
+ timelineMetric.setStartTime(startTime);
+
+ /*
+
+ 0 +30s +60s +90s +120s +150s +180s
+ | | | | | | |
+ (1) (2) (3) (4) (5) (6)
+
+ */
+ // Case 1 : Points present in all the required timeslices.
+ // Life is good! Ignore (5) and (6).
+
+ TreeMap<Long, Double> metricValues = new TreeMap<Long, Double>();
+ metricValues.put(startTime + 15*seconds, 1.0);
+ metricValues.put(startTime + 45*seconds, 2.0);
+ metricValues.put(startTime + 75*seconds, 3.0);
+ metricValues.put(startTime + 105*seconds, 4.0);
+ metricValues.put(startTime + 135*seconds, 5.0);
+ metricValues.put(startTime + 165*seconds, 6.0);
+
+ timelineMetric.setMetricValues(metricValues);
+ secondAggregator.processAggregateClusterMetrics(aggregateClusterMetrics, timelineMetric, timeslices);
+
+ Assert.assertEquals(aggregateClusterMetrics.size(), 4);
+ TimelineClusterMetric timelineClusterMetric = new TimelineClusterMetric(timelineMetric.getMetricName(),
+ timelineMetric.getAppId(), timelineMetric.getInstanceId(), startTime + 30*seconds);
+
+ Assert.assertTrue(aggregateClusterMetrics.containsKey(timelineClusterMetric));
+ Assert.assertEquals(aggregateClusterMetrics.get(timelineClusterMetric).getSum(), 1.0);
+
+ timelineClusterMetric.setTimestamp(startTime + 4*30*seconds);
+ Assert.assertTrue(aggregateClusterMetrics.containsKey(timelineClusterMetric));
+ Assert.assertEquals(aggregateClusterMetrics.get(timelineClusterMetric).getSum(),4.0);
+
+ metricValues.clear();
+ aggregateClusterMetrics.clear();
+
+ /*
+
+ 0 +30s +60s +90s +120s +150s +180s
+ | | | | | | |
+ (1) (3) (4) (5) (6)
+
+ */
+ // Case 2 : Some "middle" point missing in the required timeslices.
+ // Interpolate the middle point. Ignore (5) and (6).
+ metricValues.put(startTime + 15*seconds, 1.0);
+ metricValues.put(startTime + 75*seconds, 3.0);
+ metricValues.put(startTime + 105*seconds, 4.0);
+ metricValues.put(startTime + 135*seconds, 5.0);
+ metricValues.put(startTime + 165*seconds, 6.0);
+
+ timelineMetric.setMetricValues(metricValues);
+ secondAggregator.processAggregateClusterMetrics(aggregateClusterMetrics, timelineMetric, timeslices);
+
+ Assert.assertEquals(aggregateClusterMetrics.size(), 4);
+ timelineClusterMetric.setTimestamp(startTime + 60*seconds);
+ Assert.assertTrue(aggregateClusterMetrics.containsKey(timelineClusterMetric));
+ Assert.assertEquals(aggregateClusterMetrics.get(timelineClusterMetric).getSum(), 2.0);
+
+ metricValues.clear();
+ aggregateClusterMetrics.clear();
+
+
+ /*
+
+ 0 +30s +60s +90s +120s +150s +180s
+ | | | | | | |
+ (1) (2) (3) (5) (6)
+
+ */
+ // Case 3 : "end" point missing in the required timeslices.
+ // Use all points to get missing point if COUNTER. Else use just (3). Ignore (6).
+ metricValues.put(startTime + 15*seconds, 1.0);
+ metricValues.put(startTime + 45*seconds, 2.0);
+ metricValues.put(startTime + 75*seconds, 3.0);
+ metricValues.put(startTime + 135*seconds, 5.0);
+ metricValues.put(startTime + 165*seconds, 6.0);
+
+ timelineMetric.setMetricValues(metricValues);
+ timelineMetric.setType("GUAGE");
+ secondAggregator.processAggregateClusterMetrics(aggregateClusterMetrics, timelineMetric, timeslices);
+
+ Assert.assertEquals(aggregateClusterMetrics.size(), 4);
+ timelineClusterMetric.setTimestamp(startTime + 120*seconds);
+ Assert.assertTrue(aggregateClusterMetrics.containsKey(timelineClusterMetric));
+ Assert.assertEquals(aggregateClusterMetrics.get(timelineClusterMetric).getSum(), 3.0);
+
+ aggregateClusterMetrics.clear();
+
+ timelineMetric.setType("COUNTER");
+ secondAggregator.processAggregateClusterMetrics(aggregateClusterMetrics, timelineMetric, timeslices);
+
+ Assert.assertEquals(aggregateClusterMetrics.size(), 4);
+ timelineClusterMetric.setTimestamp(startTime + 120*seconds);
+ Assert.assertTrue(aggregateClusterMetrics.containsKey(timelineClusterMetric));
+ Assert.assertEquals(aggregateClusterMetrics.get(timelineClusterMetric).getSum(), 4.5);
+
+ metricValues.clear();
+ aggregateClusterMetrics.clear();
+
+ /*
+
+ 0 +30s +60s +90s +120s +150s +180s
+ | | | | | | |
+ (2) (3) (4) (5) (6)
+
+ */
+ // Case 4 : "start" point missing in the required timeslices.
+ // Interpolate with only (2) to get missing point if GUAGE metric. Else use all points for COUNTER.
+
+ metricValues.put(startTime + 45*seconds, 2.0);
+ metricValues.put(startTime + 75*seconds, 3.0);
+ metricValues.put(startTime + 105*seconds, 4.0);
+ metricValues.put(startTime + 135*seconds, 5.0);
+ metricValues.put(startTime + 165*seconds, 6.0);
+
+ timelineMetric.setMetricValues(metricValues);
+ timelineMetric.setType("GUAGE");
+ secondAggregator.processAggregateClusterMetrics(aggregateClusterMetrics, timelineMetric, timeslices);
+
+ Assert.assertEquals(aggregateClusterMetrics.size(), 4);
+ timelineClusterMetric.setTimestamp(startTime + 30*seconds);
+ Assert.assertTrue(aggregateClusterMetrics.containsKey(timelineClusterMetric));
+ Assert.assertEquals(aggregateClusterMetrics.get(timelineClusterMetric).getSum(), 2.0);
+
+ aggregateClusterMetrics.clear();
+
+ timelineMetric.setType("COUNTER");
+ int liveHosts = secondAggregator.processAggregateClusterMetrics(aggregateClusterMetrics, timelineMetric, timeslices);
+
+ Assert.assertEquals(liveHosts, 1);
+ Assert.assertEquals(aggregateClusterMetrics.size(), 4);
+ timelineClusterMetric.setTimestamp(startTime + 30*seconds);
+ Assert.assertTrue(aggregateClusterMetrics.containsKey(timelineClusterMetric));
+ Assert.assertEquals(aggregateClusterMetrics.get(timelineClusterMetric).getSum(), 1.5);
+
+ metricValues.clear();
+ aggregateClusterMetrics.clear();
+
+ /*
+
+ 0 +30s +60s +90s +120s +150s +180s
+ | | | | | | |
+ (5) (6)
+
+ */
+ // Case 5 : Well, we have nothing in the 2 min window.
+ // Use (5) to paint the 2 min window as (5).
+
+ metricValues.put(startTime + 135*seconds, 5.0);
+ metricValues.put(startTime + 165*seconds, 6.0);
+
+ timelineMetric.setMetricValues(metricValues);
+ timelineMetric.setType("GUAGE");
+ secondAggregator.processAggregateClusterMetrics(aggregateClusterMetrics, timelineMetric, timeslices);
+
+ Assert.assertEquals(aggregateClusterMetrics.size(), 4);
+ timelineClusterMetric.setTimestamp(startTime + 30*seconds);
+ Assert.assertTrue(aggregateClusterMetrics.containsKey(timelineClusterMetric));
+ Assert.assertEquals(aggregateClusterMetrics.get(timelineClusterMetric).getSum(), 5.0);
+
+ aggregateClusterMetrics.clear();
+
+ timelineMetric.setType("COUNTER");
+ liveHosts = secondAggregator.processAggregateClusterMetrics(aggregateClusterMetrics, timelineMetric, timeslices);
+
+ Assert.assertEquals(liveHosts, 1);
+ Assert.assertEquals(aggregateClusterMetrics.size(), 4);
+ timelineClusterMetric.setTimestamp(startTime + 60*seconds);
+ Assert.assertTrue(aggregateClusterMetrics.containsKey(timelineClusterMetric));
+ Assert.assertEquals(aggregateClusterMetrics.get(timelineClusterMetric).getSum(), 2.5);
+
+ metricValues.clear();
+ aggregateClusterMetrics.clear();
+
+ }
+
+ @Test
+ public void testLiveHostCounterMetrics() throws Exception {
+ long aggregatorInterval = 120000;
+ long sliceInterval = 30000;
+
+ Configuration configuration = new Configuration();
+ TimelineMetricMetadataManager metricMetadataManagerMock = createNiceMock(TimelineMetricMetadataManager.class);
+
+ expect(metricMetadataManagerMock.getMetadataCacheValue((TimelineMetricMetadataKey) anyObject())).andReturn(null).anyTimes();
+
+ /*
+ m1-h1-a1
+ m2-h1-a1
+ m2-h1-a2
+ m2-h2-a1
+ m2-h2-a2
+ m2-h3-a2
+
+ So live_hosts : a1 = 2
+ live_hosts : a2 = 3
+ */
+
+ TimelineMetric metric1 = new TimelineMetric("m1", "h1", "a1", null);
+ TimelineMetric metric2 = new TimelineMetric("m2", "h1", "a1", null);
+ TimelineMetric metric3 = new TimelineMetric("m2", "h1", "a2", null);
+ TimelineMetric metric4 = new TimelineMetric("m2", "h2", "a1", null);
+ TimelineMetric metric5 = new TimelineMetric("m2", "h2", "a2", null);
+ TimelineMetric metric6 = new TimelineMetric("m2", "h3", "a2", null);
+
+ expect(metricMetadataManagerMock.getMetricFromUuid((byte[]) anyObject())).
+ andReturn(metric1).andReturn(metric2).andReturn(metric3).
+ andReturn(metric4).andReturn(metric5).andReturn(metric6);
+ replay(metricMetadataManagerMock);
+
+ TimelineMetricClusterAggregatorSecond secondAggregator = new TimelineMetricClusterAggregatorSecond(
+ METRIC_AGGREGATE_SECOND, metricMetadataManagerMock, null, configuration, null,
+ aggregatorInterval, 2, "false", "", "", aggregatorInterval,
+ sliceInterval, null);
+
+ long now = System.currentTimeMillis();
+ long startTime = now - 120000;
+ long seconds = 1000;
+ List<Long[]> slices = getTimeSlices(startTime, now, sliceInterval);
+ ResultSet rs = createNiceMock(ResultSet.class);
+
+ TreeMap<Long, Double> metricValues = new TreeMap<>();
+ metricValues.put(startTime + 15 * seconds, 1.0);
+ metricValues.put(startTime + 45 * seconds, 2.0);
+ metricValues.put(startTime + 75 * seconds, 3.0);
+ metricValues.put(startTime + 105 * seconds, 4.0);
+
+ expect(rs.next()).andReturn(true).times(6);
+ expect(rs.next()).andReturn(false);
+
+ expect(rs.getLong("SERVER_TIME")).andReturn(now - 150000).times(6);
+ expect(rs.getLong("START_TIME")).andReturn(now - 150000).times(6);
+
+ ObjectMapper mapper = new ObjectMapper();
+ expect(rs.getString("METRICS")).andReturn(mapper.writeValueAsString(metricValues)).times(6);
+
+ replay(rs);
+
+ Map<TimelineClusterMetric, MetricClusterAggregate> aggregates = secondAggregator.aggregateMetricsFromResultSet(rs, slices);
+
+ Assert.assertNotNull(aggregates);
+
+ MetricClusterAggregate a1 = null, a2 = null;
+
+ for (Map.Entry<TimelineClusterMetric, MetricClusterAggregate> m : aggregates.entrySet()) {
+ if (m.getKey().getMetricName().equals("live_hosts") && m.getKey().getAppId().equals("a1")) {
+ a1 = m.getValue();
+ }
+ if (m.getKey().getMetricName().equals("live_hosts") && m.getKey().getAppId().equals("a2")) {
+ a2 = m.getValue();
+ }
+ }
+
+ Assert.assertNotNull(a1);
+ Assert.assertNotNull(a2);
+ Assert.assertEquals(2d, a1.getSum());
+ Assert.assertEquals(3d, a2.getSum());
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/timeline/aggregators/TimelineMetricClusterAggregatorSecondWithCacheSourceTest.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/timeline/aggregators/TimelineMetricClusterAggregatorSecondWithCacheSourceTest.java b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/timeline/aggregators/TimelineMetricClusterAggregatorSecondWithCacheSourceTest.java
new file mode 100644
index 0000000..34d470c
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/timeline/aggregators/TimelineMetricClusterAggregatorSecondWithCacheSourceTest.java
@@ -0,0 +1,114 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.metrics.core.timeline.aggregators;
+
+import junit.framework.Assert;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.metrics2.sink.timeline.MetricClusterAggregate;
+import org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguration;
+import org.apache.ambari.metrics.core.timeline.TimelineMetricsIgniteCache;
+import org.apache.ambari.metrics.core.timeline.discovery.TimelineMetricMetadataKey;
+import org.apache.ambari.metrics.core.timeline.discovery.TimelineMetricMetadataManager;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.powermock.core.classloader.annotations.PowerMockIgnore;
+import org.powermock.core.classloader.annotations.PrepareForTest;
+import org.powermock.modules.junit4.PowerMockRunner;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import static org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguration.TIMELINE_METRICS_COLLECTOR_IGNITE_NODES;
+import static org.apache.ambari.metrics.core.timeline.aggregators.AggregatorUtils.getTimeSlices;
+import static org.apache.ambari.metrics.core.timeline.availability.AggregationTaskRunner.AGGREGATOR_NAME.METRIC_AGGREGATE_SECOND;
+import static org.easymock.EasyMock.anyObject;
+import static org.easymock.EasyMock.createNiceMock;
+import static org.easymock.EasyMock.expect;
+import static org.easymock.EasyMock.replay;
+import static org.powermock.api.easymock.PowerMock.mockStatic;
+import static org.powermock.api.easymock.PowerMock.replayAll;
+
+@RunWith(PowerMockRunner.class)
+@PrepareForTest(TimelineMetricConfiguration.class)
+
+@PowerMockIgnore("javax.management.*")
+public class TimelineMetricClusterAggregatorSecondWithCacheSourceTest {
+
+ private static TimelineMetricsIgniteCache timelineMetricsIgniteCache;
+ @BeforeClass
+ public static void setupConf() throws Exception {
+ TimelineMetricConfiguration conf = new TimelineMetricConfiguration(new
+ Configuration(), new Configuration());
+ mockStatic(TimelineMetricConfiguration.class);
+ expect(TimelineMetricConfiguration.getInstance()).andReturn(conf).anyTimes();
+ conf.getMetricsConf().set(TIMELINE_METRICS_COLLECTOR_IGNITE_NODES, "localhost");
+ replayAll();
+
+ timelineMetricsIgniteCache = new TimelineMetricsIgniteCache();
+ }
+
+ @Test
+ public void testLiveHostCounterMetrics() throws Exception {
+ long aggregatorInterval = 120000;
+ long sliceInterval = 30000;
+
+ Configuration configuration = new Configuration();
+
+ TimelineMetricMetadataManager metricMetadataManagerMock = createNiceMock(TimelineMetricMetadataManager.class);
+ expect(metricMetadataManagerMock.getMetadataCacheValue((TimelineMetricMetadataKey) anyObject())).andReturn(null).anyTimes();
+ replay(metricMetadataManagerMock);
+
+ TimelineMetricClusterAggregatorSecondWithCacheSource secondAggregator = new TimelineMetricClusterAggregatorSecondWithCacheSource(
+ METRIC_AGGREGATE_SECOND, metricMetadataManagerMock, null, configuration, null,
+ aggregatorInterval, 2, "false", "", "", aggregatorInterval,
+ sliceInterval, null, timelineMetricsIgniteCache);
+
+ long now = System.currentTimeMillis();
+ long startTime = now - 120000;
+ long seconds = 1000;
+
+ Map<TimelineClusterMetric, MetricClusterAggregate> metricsFromCache = new HashMap<>();
+ metricsFromCache.put(new TimelineClusterMetric("m1", "a1", "i1",startTime + 15 * seconds),
+ new MetricClusterAggregate(1.0, 2, 1.0, 1.0, 1.0));
+ metricsFromCache.put(new TimelineClusterMetric("m2", "a2", "i1",startTime + 18 * seconds),
+ new MetricClusterAggregate(1.0, 5, 1.0, 1.0, 1.0));
+
+ List<Long[]> timeslices = getTimeSlices(startTime, startTime + 120*seconds, 30*seconds);
+ Map<TimelineClusterMetric, MetricClusterAggregate> aggregates = secondAggregator.aggregateMetricsFromMetricClusterAggregates(metricsFromCache, timeslices);
+
+ Assert.assertNotNull(aggregates);
+
+ MetricClusterAggregate a1 = null, a2 = null;
+
+ for (Map.Entry<TimelineClusterMetric, MetricClusterAggregate> m : aggregates.entrySet()) {
+ if (m.getKey().getMetricName().equals("live_hosts") && m.getKey().getAppId().equals("a1")) {
+ a1 = m.getValue();
+ }
+ if (m.getKey().getMetricName().equals("live_hosts") && m.getKey().getAppId().equals("a2")) {
+ a2 = m.getValue();
+ }
+ }
+
+ Assert.assertNotNull(a1);
+ Assert.assertNotNull(a2);
+ Assert.assertEquals(2d, a1.getSum());
+ Assert.assertEquals(5d, a2.getSum());
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/timeline/availability/MetricCollectorHAControllerTest.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/timeline/availability/MetricCollectorHAControllerTest.java b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/timeline/availability/MetricCollectorHAControllerTest.java
new file mode 100644
index 0000000..385a5a1
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/timeline/availability/MetricCollectorHAControllerTest.java
@@ -0,0 +1,106 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.metrics.core.timeline.availability;
+
+import junit.framework.Assert;
+import org.apache.ambari.metrics.core.timeline.AbstractMiniHBaseClusterTest;
+import org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguration;
+import org.apache.helix.HelixManager;
+import org.apache.helix.HelixManagerFactory;
+import org.apache.helix.InstanceType;
+import org.apache.helix.model.ExternalView;
+import org.apache.helix.model.InstanceConfig;
+import org.junit.Before;
+import org.junit.Test;
+import java.util.HashMap;
+import java.util.Map;
+import static org.apache.ambari.metrics.core.timeline.availability.MetricCollectorHAController.DEFAULT_STATE_MODEL;
+import static org.apache.ambari.metrics.core.timeline.availability.MetricCollectorHAController.METRIC_AGGREGATORS;
+import static org.apache.ambari.metrics.core.timeline.availability.MetricCollectorHAController.CLUSTER_NAME;
+import static org.easymock.EasyMock.createNiceMock;
+import static org.easymock.EasyMock.expect;
+import static org.easymock.EasyMock.replay;
+
+public class MetricCollectorHAControllerTest extends AbstractMiniHBaseClusterTest {
+ TimelineMetricConfiguration configuration;
+
+ @Before
+ public void setup() throws Exception {
+ configuration = createNiceMock(TimelineMetricConfiguration.class);
+
+ expect(configuration.getInstanceHostnameFromEnv()).andReturn("h1");
+ expect(configuration.getInstancePort()).andReturn("12000");
+ // jdbc:phoenix:localhost:52887:/hbase;test=true
+ String zkUrl = getUrl();
+ String port = zkUrl.split(":")[3];
+ String quorum = zkUrl.split(":")[2];
+
+ expect(configuration.getClusterZKClientPort()).andReturn(port);
+ expect(configuration.getClusterZKQuorum()).andReturn(quorum);
+ expect(configuration.getZkConnectionUrl(port, quorum)).andReturn(quorum + ":" + port);
+
+ replay(configuration);
+ }
+
+ @Test(timeout = 180000)
+ public void testHAControllerDistributedAggregation() throws Exception {
+ MetricCollectorHAController haController = new MetricCollectorHAController(configuration);
+ haController.initializeHAController();
+ // Wait for task assignment
+ Thread.sleep(10000);
+
+ Assert.assertTrue(haController.isInitialized());
+ Assert.assertEquals(1, haController.getLiveInstanceHostNames().size());
+ Assert.assertTrue(haController.getAggregationTaskRunner().performsClusterAggregation());
+ Assert.assertTrue(haController.getAggregationTaskRunner().performsHostAggregation());
+
+ // Add new instance
+ InstanceConfig instanceConfig2 = new InstanceConfig("h2_12001");
+ haController.admin.addInstance(CLUSTER_NAME, instanceConfig2);
+ HelixManager manager2 = HelixManagerFactory.getZKHelixManager(CLUSTER_NAME,
+ instanceConfig2.getInstanceName(),
+ InstanceType.PARTICIPANT, haController.zkConnectUrl);
+ manager2.getStateMachineEngine().registerStateModelFactory(DEFAULT_STATE_MODEL,
+ new OnlineOfflineStateModelFactory(instanceConfig2.getInstanceName(),
+ new AggregationTaskRunner(instanceConfig2.getInstanceName(), "", CLUSTER_NAME)));
+ manager2.connect();
+ haController.admin.rebalance(CLUSTER_NAME, METRIC_AGGREGATORS, 1);
+
+ // Wait on re-assignment of partitions
+ Thread.sleep(10000);
+ Assert.assertEquals(2, haController.getLiveInstanceHostNames().size());
+
+ ExternalView view = haController.admin.getResourceExternalView(CLUSTER_NAME, METRIC_AGGREGATORS);
+
+ Map<String, String> partitionInstanceMap = new HashMap<>();
+
+ for (String partition : view.getPartitionSet()) {
+ Map<String, String> states = view.getStateMap(partition);
+ // (instance, state) pairs
+ for (Map.Entry<String, String> stateEntry : states.entrySet()) {
+ partitionInstanceMap.put(partition, stateEntry.getKey());
+ Assert.assertEquals("ONLINE", stateEntry.getValue());
+ }
+ }
+ // Re-assigned partitions
+ Assert.assertEquals(2, partitionInstanceMap.size());
+
+ haController.getAggregationTaskRunner().stop();
+ haController.manager.disconnect();
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/timeline/discovery/TestMetadataManager.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/timeline/discovery/TestMetadataManager.java b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/timeline/discovery/TestMetadataManager.java
new file mode 100644
index 0000000..94fbb30
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/timeline/discovery/TestMetadataManager.java
@@ -0,0 +1,234 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.metrics.core.timeline.discovery;
+
+import static org.easymock.EasyMock.createNiceMock;
+import static org.easymock.EasyMock.expect;
+import static org.easymock.EasyMock.replay;
+
+import junit.framework.Assert;
+import org.apache.ambari.metrics.core.timeline.AbstractMiniHBaseClusterTest;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.metrics2.sink.timeline.TimelineMetric;
+import org.apache.hadoop.metrics2.sink.timeline.TimelineMetricMetadata;
+import org.apache.hadoop.metrics2.sink.timeline.TimelineMetrics;
+import org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguration;
+import org.apache.ambari.metrics.core.timeline.aggregators.TimelineClusterMetric;
+import org.easymock.EasyMock;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.net.URISyntaxException;
+import java.sql.SQLException;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeMap;
+
+public class TestMetadataManager extends AbstractMiniHBaseClusterTest {
+ TimelineMetricMetadataManager metadataManager;
+
+ @Before
+ public void insertDummyRecords() throws IOException, SQLException, URISyntaxException {
+
+ final long now = System.currentTimeMillis();
+
+ TimelineMetrics timelineMetrics = new TimelineMetrics();
+ TimelineMetric metric1 = new TimelineMetric();
+ metric1.setMetricName("dummy_metric1");
+ metric1.setHostName("dummy_host1");
+ metric1.setStartTime(now - 1000);
+ metric1.setAppId("dummy_app1");
+ metric1.setType("Integer");
+ metric1.setMetricValues(new TreeMap<Long, Double>() {{
+ put(now - 100, 1.0);
+ put(now - 200, 2.0);
+ put(now - 300, 3.0);
+ }});
+ timelineMetrics.getMetrics().add(metric1);
+ TimelineMetric metric2 = new TimelineMetric();
+ metric2.setMetricName("dummy_metric2");
+ metric2.setHostName("dummy_host2");
+ metric2.setStartTime(now - 1000);
+ metric2.setAppId("dummy_app2");
+ metric2.setType("Integer");
+ metric2.setInstanceId("instance2");
+ metric2.setMetricValues(new TreeMap<Long, Double>() {{
+ put(now - 100, 1.0);
+ put(now - 200, 2.0);
+ put(now - 300, 3.0);
+ }});
+ timelineMetrics.getMetrics().add(metric2);
+
+ Configuration metricsConf = createNiceMock(Configuration.class);
+ expect(metricsConf.get("timeline.metrics.service.operation.mode")).andReturn("distributed").anyTimes();
+ replay(metricsConf);
+
+ // Initialize new manager
+ metadataManager = new TimelineMetricMetadataManager(metricsConf, hdb);
+ hdb.setMetadataInstance(metadataManager);
+
+ hdb.insertMetricRecordsWithMetadata(metadataManager, timelineMetrics, true);
+ }
+
+ @Test(timeout = 180000)
+ public void testSaveMetricsMetadata() throws Exception {
+ Map<TimelineMetricMetadataKey, TimelineMetricMetadata> cachedData = metadataManager.getMetadataCache();
+
+ Assert.assertNotNull(cachedData);
+ Assert.assertEquals(2, cachedData.size());
+ TimelineMetricMetadataKey key1 = new TimelineMetricMetadataKey("dummy_metric1", "dummy_app1", null);
+ TimelineMetricMetadataKey key2 = new TimelineMetricMetadataKey("dummy_metric2", "dummy_app2", "instance2");
+ TimelineMetricMetadata value1 = new TimelineMetricMetadata("dummy_metric1",
+ "dummy_app1", null, null, "Integer", 1L, true, true);
+ TimelineMetricMetadata value2 = new TimelineMetricMetadata("dummy_metric2",
+ "dummy_app2", "instance2", null, "Integer", 1L, true, true);
+
+ Assert.assertEquals(value1, cachedData.get(key1));
+ Assert.assertEquals(value2, cachedData.get(key2));
+
+ TimelineMetricMetadataSync syncRunnable = new TimelineMetricMetadataSync(metadataManager);
+ syncRunnable.run();
+
+ Map<TimelineMetricMetadataKey, TimelineMetricMetadata> savedData =
+ hdb.getTimelineMetricMetadata();
+
+ Assert.assertNotNull(savedData);
+ Assert.assertEquals(2, savedData.size());
+ Assert.assertEquals(value1, savedData.get(key1));
+ Assert.assertEquals(value2, savedData.get(key2));
+
+ Map<String, TimelineMetricHostMetadata> cachedHostData = metadataManager.getHostedAppsCache();
+ Map<String, TimelineMetricHostMetadata> savedHostData = metadataManager.getHostedAppsFromStore();
+ Assert.assertEquals(cachedData.size(), savedData.size());
+ Assert.assertEquals("dummy_app1", cachedHostData.get("dummy_host1").getHostedApps().keySet().iterator().next());
+ Assert.assertEquals("dummy_app2", cachedHostData.get("dummy_host2").getHostedApps().keySet().iterator().next());
+ Assert.assertEquals("dummy_app1", savedHostData.get("dummy_host1").getHostedApps().keySet().iterator().next());
+ Assert.assertEquals("dummy_app2", savedHostData.get("dummy_host2").getHostedApps().keySet().iterator().next());
+
+ Map<String, Set<String>> cachedHostInstanceData = metadataManager.getHostedInstanceCache();
+ Map<String, Set<String>> savedHostInstanceData = metadataManager.getHostedInstancesFromStore();
+ Assert.assertEquals(cachedHostInstanceData.size(), savedHostInstanceData.size());
+ Assert.assertEquals("dummy_host2", cachedHostInstanceData.get("instance2").iterator().next());
+ }
+
+ @Test
+ public void testGenerateUuidFromMetric() throws SQLException {
+
+ TimelineMetric timelineMetric = new TimelineMetric();
+ timelineMetric.setMetricName("regionserver.Server.blockCacheExpressHitPercent");
+ timelineMetric.setAppId("hbase");
+ timelineMetric.setHostName("avijayan-ams-2.openstacklocal");
+ timelineMetric.setInstanceId("test1");
+
+ byte[] uuid = metadataManager.getUuid(timelineMetric);
+ Assert.assertNotNull(uuid);
+ Assert.assertEquals(uuid.length, 20);
+
+ byte[] uuidWithoutHost = metadataManager.getUuid(new TimelineClusterMetric(timelineMetric.getMetricName(), timelineMetric.getAppId(), timelineMetric.getInstanceId(), -1));
+ Assert.assertNotNull(uuidWithoutHost);
+ Assert.assertEquals(uuidWithoutHost.length, 16);
+
+ TimelineMetric metric2 = metadataManager.getMetricFromUuid(uuid);
+ Assert.assertEquals(metric2, timelineMetric);
+ TimelineMetric metric3 = metadataManager.getMetricFromUuid(uuidWithoutHost);
+ Assert.assertEquals(metric3.getMetricName(), timelineMetric.getMetricName());
+ Assert.assertEquals(metric3.getAppId(), timelineMetric.getAppId());
+ Assert.assertEquals(metric3.getInstanceId(), timelineMetric.getInstanceId());
+ Assert.assertEquals(metric3.getHostName(), null);
+
+ String metricName1 = metadataManager.getMetricNameFromUuid(uuid);
+ Assert.assertEquals(metricName1, "regionserver.Server.blockCacheExpressHitPercent");
+ String metricName2 = metadataManager.getMetricNameFromUuid(uuidWithoutHost);
+ Assert.assertEquals(metricName2, "regionserver.Server.blockCacheExpressHitPercent");
+ }
+
+ @Test
+ public void testWildcardSanitization() throws IOException, SQLException, URISyntaxException {
+ // Initialize new manager
+ metadataManager = new TimelineMetricMetadataManager(new Configuration(), hdb);
+ final long now = System.currentTimeMillis();
+
+ TimelineMetrics timelineMetrics = new TimelineMetrics();
+
+ TimelineMetric metric1 = new TimelineMetric();
+ metric1.setMetricName("dummy_m1");
+ metric1.setHostName("dummy_host1");
+ metric1.setStartTime(now - 1000);
+ metric1.setAppId("dummy_app1");
+ metric1.setType("Integer");
+ metric1.setMetricValues(new TreeMap<Long, Double>() {{
+ put(now - 100, 1.0);
+ put(now - 200, 2.0);
+ put(now - 300, 3.0);
+ }});
+ timelineMetrics.getMetrics().add(metric1);
+
+ TimelineMetric metric2 = new TimelineMetric();
+ metric2.setMetricName("dummy_m2");
+ metric2.setHostName("dummy_host2");
+ metric2.setStartTime(now - 1000);
+ metric2.setAppId("dummy_app2");
+ metric2.setType("Integer");
+ metric2.setMetricValues(new TreeMap<Long, Double>() {{
+ put(now - 100, 1.0);
+ put(now - 200, 2.0);
+ put(now - 300, 3.0);
+ }});
+ timelineMetrics.getMetrics().add(metric2);
+
+ TimelineMetric metric3 = new TimelineMetric();
+ metric3.setMetricName("gummy_3");
+ metric3.setHostName("dummy_3h");
+ metric3.setStartTime(now - 1000);
+ metric3.setAppId("dummy_app3");
+ metric3.setType("Integer");
+ metric3.setMetricValues(new TreeMap<Long, Double>() {{
+ put(now - 100, 1.0);
+ put(now - 200, 2.0);
+ put(now - 300, 3.0);
+ }});
+ timelineMetrics.getMetrics().add(metric3);
+
+ Configuration metricsConf = new Configuration();
+ TimelineMetricConfiguration configuration = EasyMock.createNiceMock(TimelineMetricConfiguration.class);
+ expect(configuration.getMetricsConf()).andReturn(metricsConf).once();
+ replay(configuration);
+
+ hdb.insertMetricRecordsWithMetadata(metadataManager, timelineMetrics, true);
+
+ List<byte[]> uuids = metadataManager.getUuids(Collections.singletonList("dummy_m%"),
+ Collections.singletonList("dummy_host2"), "dummy_app1", null);
+ Assert.assertTrue(uuids.size() == 2);
+
+ uuids = metadataManager.getUuids(Collections.singletonList("dummy_m%"),
+ Collections.singletonList("dummy_host%"), "dummy_app2", null);
+ Assert.assertTrue(uuids.size() == 4);
+
+ Collection<String> metrics = Arrays.asList("dummy_m%", "dummy_3", "dummy_m2");
+ List<String> hosts = Arrays.asList("dummy_host%", "dummy_3h");
+ uuids = metadataManager.getUuids(metrics, hosts, "dummy_app2", null);
+ Assert.assertTrue(uuids.size() == 9);
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/timeline/discovery/TestMetadataSync.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/timeline/discovery/TestMetadataSync.java b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/timeline/discovery/TestMetadataSync.java
new file mode 100644
index 0000000..80eb89e
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/timeline/discovery/TestMetadataSync.java
@@ -0,0 +1,126 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.metrics.core.timeline.discovery;
+
+import junit.framework.Assert;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.metrics2.sink.timeline.TimelineMetricMetadata;
+import org.apache.ambari.metrics.core.timeline.PhoenixHBaseAccessor;
+import org.junit.Test;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import static org.apache.hadoop.metrics2.sink.timeline.TimelineMetricMetadata.MetricType.GAUGE;
+import static org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguration.TIMELINE_METRIC_METADATA_FILTERS;
+import static org.easymock.EasyMock.createNiceMock;
+import static org.easymock.EasyMock.expect;
+import static org.easymock.EasyMock.replay;
+import static org.easymock.EasyMock.verify;
+
+public class TestMetadataSync {
+ @Test
+ public void testRefreshMetadataOnWrite() throws Exception {
+ Configuration configuration = createNiceMock(Configuration.class);
+ PhoenixHBaseAccessor hBaseAccessor = createNiceMock(PhoenixHBaseAccessor.class);
+
+ final TimelineMetricMetadata testMetadata1 = new TimelineMetricMetadata(
+ "m1", "a1", null, "", GAUGE.name(), System.currentTimeMillis(), true, false);
+ final TimelineMetricMetadata testMetadata2 = new TimelineMetricMetadata(
+ "m2", "a2", null, "", GAUGE.name(), System.currentTimeMillis(), true, false);
+
+ Map<TimelineMetricMetadataKey, TimelineMetricMetadata> metadata =
+ new HashMap<TimelineMetricMetadataKey, TimelineMetricMetadata>() {{
+ put(new TimelineMetricMetadataKey("m1", "a1", null), testMetadata1);
+ put(new TimelineMetricMetadataKey("m2", "a2", null), testMetadata2);
+ }};
+
+ Map<String, TimelineMetricHostMetadata> hostedApps = new HashMap<String, TimelineMetricHostMetadata>() {{
+ put("h1", new TimelineMetricHostMetadata(new HashSet<>(Arrays.asList("a1"))));
+ put("h2", new TimelineMetricHostMetadata((new HashSet<>(Arrays.asList("a1", "a2")))));
+ }};
+
+ Map<String, Set<String>> hostedInstances = new HashMap<String, Set<String>>() {{
+ put("i1", new HashSet<>(Arrays.asList("h1")));
+ put("i2", new HashSet<>(Arrays.asList("h1", "h2")));
+ }};
+
+ expect(configuration.get("timeline.metrics.service.operation.mode")).andReturn("distributed");
+ expect(hBaseAccessor.getTimelineMetricMetadata()).andReturn(metadata);
+ expect(hBaseAccessor.getHostedAppsMetadata()).andReturn(hostedApps);
+ expect(hBaseAccessor.getInstanceHostsMetdata()).andReturn(hostedInstances);
+
+ replay(configuration, hBaseAccessor);
+
+ TimelineMetricMetadataManager metadataManager = new TimelineMetricMetadataManager(configuration, hBaseAccessor);
+
+ metadataManager.metricMetadataSync = new TimelineMetricMetadataSync(metadataManager);
+
+ metadataManager.metricMetadataSync.run();
+
+ verify(configuration, hBaseAccessor);
+
+ metadata = metadataManager.getMetadataCache();
+ Assert.assertEquals(2, metadata.size());
+ Assert.assertTrue(metadata.containsKey(new TimelineMetricMetadataKey("m1", "a1", null)));
+ Assert.assertTrue(metadata.containsKey(new TimelineMetricMetadataKey("m2", "a2", null)));
+
+ hostedApps = metadataManager.getHostedAppsCache();
+ Assert.assertEquals(2, hostedApps.size());
+ Assert.assertEquals(1, hostedApps.get("h1").getHostedApps().size());
+ Assert.assertEquals(2, hostedApps.get("h2").getHostedApps().size());
+
+ hostedInstances = metadataManager.getHostedInstanceCache();
+ Assert.assertEquals(2, hostedInstances.size());
+ Assert.assertEquals(1, hostedInstances.get("i1").size());
+ Assert.assertEquals(2, hostedInstances.get("i2").size());
+
+ }
+
+ @Test
+ public void testFilterByRegexOnMetricName() throws Exception {
+ Configuration configuration = createNiceMock(Configuration.class);
+ PhoenixHBaseAccessor hBaseAccessor = createNiceMock(PhoenixHBaseAccessor.class);
+
+ TimelineMetricMetadata metadata1 = new TimelineMetricMetadata(
+ "xxx.abc.yyy", "a1", null, "", GAUGE.name(), System.currentTimeMillis(), true, false);
+ TimelineMetricMetadata metadata2 = new TimelineMetricMetadata(
+ "xxx.cdef.yyy", "a2", null, "", GAUGE.name(), System.currentTimeMillis(), true, false);
+ TimelineMetricMetadata metadata3 = new TimelineMetricMetadata(
+ "xxx.pqr.zzz", "a3", null, "", GAUGE.name(), System.currentTimeMillis(), true, false);
+
+ expect(configuration.get(TIMELINE_METRIC_METADATA_FILTERS)).andReturn("abc,cde");
+
+ replay(configuration, hBaseAccessor);
+
+ TimelineMetricMetadataManager metadataManager = new TimelineMetricMetadataManager(configuration, hBaseAccessor);
+
+ metadataManager.putIfModifiedTimelineMetricMetadata(metadata1);
+ metadataManager.putIfModifiedTimelineMetricMetadata(metadata2);
+ metadataManager.putIfModifiedTimelineMetricMetadata(metadata3);
+
+ verify(configuration, hBaseAccessor);
+
+ Map<TimelineMetricMetadataKey, TimelineMetricMetadata> metadata =
+ metadataManager.getMetadataCache();
+
+ Assert.assertEquals(1, metadata.size());
+ Assert.assertEquals("xxx.pqr.zzz", metadata.keySet().iterator().next().getMetricName());
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/timeline/function/TimelineMetricsSeriesAggregateFunctionTest.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/timeline/function/TimelineMetricsSeriesAggregateFunctionTest.java b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/timeline/function/TimelineMetricsSeriesAggregateFunctionTest.java
new file mode 100644
index 0000000..492d5a0
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/timeline/function/TimelineMetricsSeriesAggregateFunctionTest.java
@@ -0,0 +1,188 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.metrics.core.timeline.function;
+
+import com.google.common.collect.Lists;
+import org.apache.hadoop.metrics2.sink.timeline.TimelineMetric;
+import org.apache.hadoop.metrics2.sink.timeline.TimelineMetrics;
+import org.junit.Test;
+
+import java.util.Map;
+import java.util.TreeMap;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+public class TimelineMetricsSeriesAggregateFunctionTest {
+
+ public static final double DELTA = 0.00000000001;
+
+ @Test public void testSeriesAggregateBySummation() throws Exception {
+ TimelineMetrics testMetrics = getTestObject();
+ // all TimelineMetric are having same values
+ TreeMap<Long, Double> metricValues = testMetrics.getMetrics().get(0).getMetricValues();
+
+ TimelineMetricsSeriesAggregateFunction function = TimelineMetricsSeriesAggregateFunctionFactory
+ .newInstance(SeriesAggregateFunction.SUM);
+ TimelineMetric aggregatedMetric = function.apply(testMetrics);
+
+ String aggregatedMetricName = aggregatedMetric.getMetricName();
+ String aggregatedHostName = aggregatedMetric.getHostName();
+ String aggregatedAppId = aggregatedMetric.getAppId();
+ String aggregatedInstanceId = aggregatedMetric.getInstanceId();
+
+ for (TimelineMetric testMetric : testMetrics.getMetrics()) {
+ assertTrue(aggregatedMetricName.contains(testMetric.getMetricName()));
+ assertTrue(aggregatedHostName.contains(testMetric.getHostName()));
+ if (!testMetric.getMetricName().equals("byte_in.3")) {
+ assertTrue(aggregatedAppId.contains(testMetric.getAppId()));
+ assertTrue(aggregatedInstanceId.contains(testMetric.getInstanceId()));
+ }
+ }
+
+ TreeMap<Long, Double> summationMetricValues = aggregatedMetric.getMetricValues();
+ assertEquals(3, summationMetricValues.size());
+ for (Map.Entry<Long, Double> tsAndValue : summationMetricValues.entrySet()) {
+ assertEquals(metricValues.get(tsAndValue.getKey()) * 3, tsAndValue.getValue(), DELTA);
+ }
+ }
+
+ @Test public void testSeriesAggregateByAverage() throws Exception {
+ TimelineMetrics testMetrics = getTestObject();
+ // all TimelineMetric are having same values
+ TreeMap<Long, Double> metricValues = testMetrics.getMetrics().get(0).getMetricValues();
+
+ TimelineMetricsSeriesAggregateFunction function = TimelineMetricsSeriesAggregateFunctionFactory
+ .newInstance(SeriesAggregateFunction.AVG);
+ TimelineMetric aggregatedMetric = function.apply(testMetrics);
+
+ // checks only values, others are covered by testSeriesAggregateBySummation
+ TreeMap<Long, Double> averageMetricValues = aggregatedMetric.getMetricValues();
+ assertEquals(3, averageMetricValues.size());
+ for (Map.Entry<Long, Double> tsAndValue : averageMetricValues.entrySet()) {
+ assertEquals(metricValues.get(tsAndValue.getKey()), tsAndValue.getValue(), DELTA);
+ }
+ }
+
+ @Test public void testSeriesAggregateByMax() throws Exception {
+ TimelineMetrics testMetrics = getTestObject();
+
+ // override metric values
+ TreeMap<Long, Double> metricValues = new TreeMap<>();
+ metricValues.put(1L, 1.0);
+ metricValues.put(2L, 2.0);
+ metricValues.put(3L, 3.0);
+
+ testMetrics.getMetrics().get(0).setMetricValues(metricValues);
+
+ TreeMap<Long, Double> metricValues2 = new TreeMap<>();
+ metricValues2.put(1L, 2.0);
+ metricValues2.put(2L, 4.0);
+ metricValues2.put(3L, 6.0);
+
+ testMetrics.getMetrics().get(1).setMetricValues(metricValues2);
+
+ TreeMap<Long, Double> metricValues3 = new TreeMap<>();
+ metricValues3.put(1L, 3.0);
+ metricValues3.put(2L, 6.0);
+ metricValues3.put(3L, 9.0);
+
+ testMetrics.getMetrics().get(2).setMetricValues(metricValues3);
+
+ TimelineMetricsSeriesAggregateFunction function = TimelineMetricsSeriesAggregateFunctionFactory
+ .newInstance(SeriesAggregateFunction.MAX);
+ TimelineMetric aggregatedMetric = function.apply(testMetrics);
+
+ // checks only values, others are covered by testSeriesAggregateBySummation
+ TreeMap<Long, Double> maxMetricValues = aggregatedMetric.getMetricValues();
+ assertEquals(3, maxMetricValues.size());
+ for (Map.Entry<Long, Double> tsAndValue : maxMetricValues.entrySet()) {
+ assertEquals(metricValues3.get(tsAndValue.getKey()), tsAndValue.getValue(), DELTA);
+ }
+ }
+
+ @Test public void testSeriesAggregateByMin() throws Exception {
+ TimelineMetrics testMetrics = getTestObject();
+
+ // override metric values
+ TreeMap<Long, Double> metricValues = new TreeMap<>();
+ metricValues.put(1L, 1.0);
+ metricValues.put(2L, 2.0);
+ metricValues.put(3L, 3.0);
+
+ testMetrics.getMetrics().get(0).setMetricValues(metricValues);
+
+ TreeMap<Long, Double> metricValues2 = new TreeMap<>();
+ metricValues2.put(1L, 2.0);
+ metricValues2.put(2L, 4.0);
+ metricValues2.put(3L, 6.0);
+
+ testMetrics.getMetrics().get(1).setMetricValues(metricValues2);
+
+ TreeMap<Long, Double> metricValues3 = new TreeMap<>();
+ metricValues3.put(1L, 3.0);
+ metricValues3.put(2L, 6.0);
+ metricValues3.put(3L, 9.0);
+
+ testMetrics.getMetrics().get(2).setMetricValues(metricValues3);
+
+ TimelineMetricsSeriesAggregateFunction function = TimelineMetricsSeriesAggregateFunctionFactory
+ .newInstance(SeriesAggregateFunction.MIN);
+ TimelineMetric aggregatedMetric = function.apply(testMetrics);
+
+ // checks only values, others are covered by testSeriesAggregateBySummation
+ TreeMap<Long, Double> minMetricValues = aggregatedMetric.getMetricValues();
+ assertEquals(3, minMetricValues.size());
+ for (Map.Entry<Long, Double> tsAndValue : minMetricValues.entrySet()) {
+ assertEquals(metricValues.get(tsAndValue.getKey()), tsAndValue.getValue(), DELTA);
+ }
+ }
+
+ private TimelineMetrics getTestObject() {
+ TimelineMetric metric = new TimelineMetric();
+ metric.setMetricName("byte_in.1");
+ metric.setHostName("host1");
+ metric.setAppId("app1");
+ metric.setInstanceId("instance1");
+
+ TimelineMetric metric2 = new TimelineMetric();
+ metric2.setMetricName("byte_in.2");
+ metric2.setHostName("host2");
+ metric2.setAppId("app2");
+ metric2.setInstanceId("instance2");
+
+ TimelineMetric metric3 = new TimelineMetric();
+ metric3.setMetricName("byte_in.3");
+ metric3.setHostName("host3");
+ // appId and instanceId for metric3 are null
+
+ TreeMap<Long, Double> metricValues = new TreeMap<>();
+ metricValues.put(1L, 3.0);
+ metricValues.put(2L, 2 * 3.0);
+ metricValues.put(3L, 3 * 3.0);
+
+ metric.setMetricValues(metricValues);
+ metric2.setMetricValues(metricValues);
+ metric3.setMetricValues(metricValues);
+
+ TimelineMetrics metrics = new TimelineMetrics();
+ metrics.setMetrics(Lists.newArrayList(metric, metric2, metric3));
+
+ return metrics;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/timeline/query/DefaultConditionTest.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/timeline/query/DefaultConditionTest.java b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/timeline/query/DefaultConditionTest.java
new file mode 100644
index 0000000..7d7e0f7
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/timeline/query/DefaultConditionTest.java
@@ -0,0 +1,170 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.metrics.core.timeline.query;
+
+import junit.framework.Assert;
+import org.apache.commons.collections.CollectionUtils;
+import org.junit.Test;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class DefaultConditionTest {
+
+ @Test
+ public void testMetricNameWhereCondition() {
+ //EMPTY
+ List<byte[]> uuids = new ArrayList<>();
+ DefaultCondition condition = new DefaultCondition(uuids,null,null,null,null,null,null,null,null,true);
+ StringBuilder sb = new StringBuilder();
+ condition.appendUuidClause(sb);
+ Assert.assertEquals(sb.toString(), "");
+ Assert.assertTrue(CollectionUtils.isEqualCollection(uuids, condition.getUuids()));
+
+ //Metric uuid
+ uuids.add(new byte[16]);
+ condition = new DefaultCondition(uuids,null,null,null,null,null,null,null,null,true);
+ sb = new StringBuilder();
+ condition.appendUuidClause(sb);
+ Assert.assertEquals(sb.toString(), "(UUID LIKE ?)");
+ Assert.assertEquals(uuids.size(), condition.getUuids().size());
+ Assert.assertTrue(new String(condition.getUuids().get(0)).endsWith("%"));
+
+ //metric uuid + Host uuid
+ uuids.add(new byte[4]);
+ condition = new DefaultCondition(uuids,null,null,null,null,null,null,null,null,true);
+ sb = new StringBuilder();
+ condition.appendUuidClause(sb);
+ Assert.assertEquals(sb.toString(), "(UUID LIKE ? AND UUID LIKE ?)");
+ Assert.assertEquals(uuids.size(), condition.getUuids().size());
+ Assert.assertTrue(new String(condition.getUuids().get(1)).startsWith("%"));
+
+ //metric + host + full
+ uuids.add(new byte[20]);
+ uuids.add(new byte[20]);
+ condition = new DefaultCondition(uuids,null,null,null,null,null,null,null,null,true);
+ sb = new StringBuilder();
+ condition.appendUuidClause(sb);
+ Assert.assertEquals(sb.toString(), "(UUID IN (?, ?) AND UUID LIKE ? AND UUID LIKE ?)");
+ Assert.assertEquals(uuids.size(), condition.getUuids().size());
+
+ //Only IN clause.
+ uuids.clear();
+ uuids.add(new byte[20]);
+ condition = new DefaultCondition(uuids,null,null,null,null,null,null,null,null,true);
+ sb = new StringBuilder();
+ condition.appendUuidClause(sb);
+ Assert.assertEquals(sb.toString(), "(UUID IN (?))");
+ Assert.assertEquals(uuids.size(), condition.getUuids().size());
+
+ //metric NOT LIKE
+ uuids.clear();
+ uuids.add(new byte[16]);
+ condition = new DefaultCondition(uuids,null,null,null,null,null,null,null,null,true);
+ sb = new StringBuilder();
+ condition.setMetricNamesNotCondition(true);
+ condition.appendUuidClause(sb);
+ Assert.assertEquals(sb.toString(), "(UUID NOT LIKE ?)");
+ Assert.assertEquals(uuids.size(), condition.getUuids().size());
+
+ //metric NOT LIKE host LIKE
+ uuids.clear();
+ uuids.add(new byte[16]);
+ uuids.add(new byte[4]);
+ condition = new DefaultCondition(uuids,null,null,null,null,null,null,null,null,true);
+ sb = new StringBuilder();
+ condition.setMetricNamesNotCondition(true);
+ condition.appendUuidClause(sb);
+ Assert.assertEquals(sb.toString(), "(UUID NOT LIKE ? AND UUID LIKE ?)");
+ Assert.assertEquals(uuids.size(), condition.getUuids().size());
+ Assert.assertTrue(new String(condition.getUuids().get(0)).endsWith("%"));
+ Assert.assertTrue(new String(condition.getUuids().get(1)).startsWith("%"));
+
+ //metric LIKE host NOT LIKE
+ uuids.clear();
+ uuids.add(new byte[16]);
+ uuids.add(new byte[4]);
+ condition = new DefaultCondition(uuids,null,null,null,null,null,null,null,null,true);
+ sb = new StringBuilder();
+ condition.setHostnamesNotCondition(true);
+ condition.appendUuidClause(sb);
+ Assert.assertEquals(sb.toString(), "(UUID LIKE ? AND UUID NOT LIKE ?)");
+ Assert.assertEquals(uuids.size(), condition.getUuids().size());
+ Assert.assertTrue(new String(condition.getUuids().get(0)).endsWith("%"));
+ Assert.assertTrue(new String(condition.getUuids().get(1)).startsWith("%"));
+
+ //metric LIKE or LIKE host LIKE
+ uuids.clear();
+ uuids.add(new byte[4]);
+ uuids.add(new byte[16]);
+ uuids.add(new byte[16]);
+ condition = new DefaultCondition(uuids,null,null,null,null,null,null,null,null,true);
+ sb = new StringBuilder();
+ condition.appendUuidClause(sb);
+ Assert.assertEquals(sb.toString(), "((UUID LIKE ? OR UUID LIKE ?) AND UUID LIKE ?)");
+ Assert.assertEquals(uuids.size(), condition.getUuids().size());
+ Assert.assertTrue(new String(condition.getUuids().get(0)).endsWith("%"));
+ Assert.assertTrue(new String(condition.getUuids().get(1)).endsWith("%"));
+ Assert.assertTrue(new String(condition.getUuids().get(2)).startsWith("%"));
+
+ //UUID in metric LIKE or LIKE host LIKE
+ uuids.clear();
+ uuids.add(new byte[16]);
+ uuids.add(new byte[16]);
+ uuids.add(new byte[20]);
+ uuids.add(new byte[4]);
+ condition = new DefaultCondition(uuids,null,null,null,null,null,null,null,null,true);
+ sb = new StringBuilder();
+ condition.appendUuidClause(sb);
+ Assert.assertEquals(sb.toString(), "(UUID IN (?) AND (UUID LIKE ? OR UUID LIKE ?) AND UUID LIKE ?)");
+ Assert.assertEquals(uuids.size(), condition.getUuids().size());
+ Assert.assertTrue(new String(condition.getUuids().get(1)).endsWith("%"));
+ Assert.assertTrue(new String(condition.getUuids().get(2)).endsWith("%"));
+ Assert.assertTrue(new String(condition.getUuids().get(3)).startsWith("%"));
+
+ //metric LIKE host LIKE or LIKE
+ uuids.clear();
+ uuids.add(new byte[16]);
+ uuids.add(new byte[4]);
+ uuids.add(new byte[4]);
+ condition = new DefaultCondition(uuids,null,null,null,null,null,null,null,null,true);
+ sb = new StringBuilder();
+ condition.appendUuidClause(sb);
+ Assert.assertEquals(sb.toString(), "(UUID LIKE ? AND (UUID LIKE ? OR UUID LIKE ?))");
+ Assert.assertEquals(uuids.size(), condition.getUuids().size());
+ Assert.assertTrue(new String(condition.getUuids().get(0)).endsWith("%"));
+ Assert.assertTrue(new String(condition.getUuids().get(1)).startsWith("%"));
+ Assert.assertTrue(new String(condition.getUuids().get(2)).startsWith("%"));
+
+ //UUID NOT IN metric LIKE host LIKE
+ uuids.clear();
+ uuids.add(new byte[20]);
+ uuids.add(new byte[16]);
+ uuids.add(new byte[4]);
+ condition = new DefaultCondition(uuids,null,null,null,null,null,null,null,null,true);
+ sb = new StringBuilder();
+ condition.setUuidNotCondition(true);
+ condition.appendUuidClause(sb);
+ Assert.assertEquals(sb.toString(), "(UUID NOT IN (?) AND UUID LIKE ? AND UUID LIKE ?)");
+ Assert.assertEquals(uuids.size(), condition.getUuids().size());
+ Assert.assertTrue(new String(condition.getUuids().get(1)).endsWith("%"));
+ Assert.assertTrue(new String(condition.getUuids().get(2)).startsWith("%"));
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/timeline/source/RawMetricsSourceTest.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/timeline/source/RawMetricsSourceTest.java b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/timeline/source/RawMetricsSourceTest.java
new file mode 100644
index 0000000..7d58682
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/timeline/source/RawMetricsSourceTest.java
@@ -0,0 +1,141 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.metrics.core.timeline.source;
+
+import static org.apache.ambari.metrics.core.timeline.source.InternalSourceProvider.SOURCE_NAME.RAW_METRICS;
+import static org.easymock.EasyMock.capture;
+import static org.easymock.EasyMock.createNiceMock;
+import static org.easymock.EasyMock.expect;
+import static org.easymock.EasyMock.expectLastCall;
+import static org.easymock.EasyMock.replay;
+import static org.easymock.EasyMock.verify;
+import static org.powermock.api.easymock.PowerMock.mockStatic;
+import static org.powermock.api.easymock.PowerMock.replayAll;
+
+import java.util.Collection;
+import java.util.Collections;
+import java.util.TreeMap;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.metrics2.sink.timeline.TimelineMetric;
+import org.apache.hadoop.metrics2.sink.timeline.TimelineMetrics;
+import org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguration;
+import org.apache.ambari.metrics.core.timeline.sink.ExternalMetricsSink;
+import org.easymock.Capture;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.powermock.core.classloader.annotations.PrepareForTest;
+import org.powermock.modules.junit4.PowerMockRunner;
+
+import junit.framework.Assert;
+
+@RunWith(PowerMockRunner.class)
+@PrepareForTest(TimelineMetricConfiguration.class)
+public class RawMetricsSourceTest {
+
+ @Before
+ public void setupConf() throws Exception {
+ TimelineMetricConfiguration conf = new TimelineMetricConfiguration(new
+ Configuration(), new Configuration());
+ mockStatic(TimelineMetricConfiguration.class);
+ expect(TimelineMetricConfiguration.getInstance()).andReturn(conf).anyTimes();
+ replayAll();
+ }
+
+ @Test
+ public void testRawMetricsSourcedAtFlushInterval() throws Exception {
+ InternalSourceProvider internalSourceProvider = new DefaultInternalMetricsSourceProvider();
+ ExternalMetricsSink rawMetricsSink = createNiceMock(ExternalMetricsSink.class);
+ expect(rawMetricsSink.getFlushSeconds()).andReturn(1);
+ expect(rawMetricsSink.getSinkTimeOutSeconds()).andReturn(1);
+ Capture<Collection<TimelineMetrics>> metricsCapture = new Capture<>();
+ rawMetricsSink.sinkMetricData(capture(metricsCapture));
+ expectLastCall();
+ replay(rawMetricsSink);
+
+ InternalMetricsSource rawMetricsSource = internalSourceProvider.getInternalMetricsSource(RAW_METRICS, 1, rawMetricsSink);
+ TimelineMetrics timelineMetrics = new TimelineMetrics();
+
+ final long now = System.currentTimeMillis();
+ TimelineMetric metric1 = new TimelineMetric();
+ metric1.setMetricName("m1");
+ metric1.setAppId("a1");
+ metric1.setInstanceId("i1");
+ metric1.setHostName("h1");
+ metric1.setStartTime(now - 200);
+ metric1.setMetricValues(new TreeMap<Long, Double>() {{
+ put(now - 100, 1.0);
+ put(now - 200, 2.0);
+ }});
+ timelineMetrics.getMetrics().add(metric1);
+
+ rawMetricsSource.publishTimelineMetrics(Collections.singletonList(timelineMetrics));
+
+ verify(rawMetricsSink);
+ }
+
+ @Test(timeout = 10000)
+ public void testRawMetricsCachedAndSourced() throws Exception {
+ ExternalMetricsSink rawMetricsSink = createNiceMock(ExternalMetricsSink.class);
+ expect(rawMetricsSink.getFlushSeconds()).andReturn(2).anyTimes();
+ expect(rawMetricsSink.getSinkTimeOutSeconds()).andReturn(1).anyTimes();
+
+ class CaptureOnce<T> extends Capture<T> {
+ @Override
+ public void setValue(T value) {
+ if (!hasCaptured()) {
+ super.setValue(value);
+ }
+ }
+ }
+ Capture<Collection<TimelineMetrics>> metricsCapture = new CaptureOnce<>();
+
+ rawMetricsSink.sinkMetricData(capture(metricsCapture));
+ expectLastCall();
+ replay(rawMetricsSink);
+
+ InternalSourceProvider internalSourceProvider = new DefaultInternalMetricsSourceProvider();
+ InternalMetricsSource rawMetricsSource = internalSourceProvider.getInternalMetricsSource(RAW_METRICS, 1, rawMetricsSink);
+ TimelineMetrics timelineMetrics = new TimelineMetrics();
+
+ final long now = System.currentTimeMillis();
+ TimelineMetric metric1 = new TimelineMetric();
+ metric1.setMetricName("m1");
+ metric1.setAppId("a1");
+ metric1.setInstanceId("i1");
+ metric1.setHostName("h1");
+ metric1.setStartTime(now - 200);
+ metric1.setMetricValues(new TreeMap<Long, Double>() {{
+ put(now - 100, 1.0);
+ put(now - 200, 2.0);
+ }});
+ timelineMetrics.getMetrics().add(metric1);
+
+ rawMetricsSource.publishTimelineMetrics(Collections.singletonList(timelineMetrics));
+
+ // Wait on eviction
+ Thread.sleep(5000);
+
+ verify(rawMetricsSink);
+
+ Assert.assertTrue(metricsCapture.hasCaptured());
+ Assert.assertTrue(metricsCapture.getValue().iterator().next().getMetrics().iterator().next().equals(metric1));
+ }
+
+}
[14/28] ambari git commit: AMBARI-22740 : Fix integration test for
HBase in branch-3.0-ams due to UUID changes. (avijayan)
Posted by av...@apache.org.
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricClusterAggregator.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricClusterAggregator.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricClusterAggregator.java
deleted file mode 100644
index 5c370f4..0000000
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricClusterAggregator.java
+++ /dev/null
@@ -1,145 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.aggregators;
-
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query.PhoenixTransactSQL.GET_CLUSTER_AGGREGATE_SQL;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query.PhoenixTransactSQL.GET_CLUSTER_AGGREGATE_TIME_SQL;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query.PhoenixTransactSQL.METRICS_CLUSTER_AGGREGATE_TABLE_NAME;
-
-import java.io.IOException;
-import java.sql.ResultSet;
-import java.sql.SQLException;
-import java.util.HashMap;
-import java.util.Map;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.metrics2.sink.timeline.MetricClusterAggregate;
-import org.apache.hadoop.metrics2.sink.timeline.MetricHostAggregate;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.PhoenixHBaseAccessor;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.availability.AggregationTaskRunner.AGGREGATOR_NAME;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.availability.MetricCollectorHAController;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.discovery.TimelineMetricMetadataManager;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query.Condition;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query.DefaultCondition;
-
-public class TimelineMetricClusterAggregator extends AbstractTimelineAggregator {
- private final TimelineMetricReadHelper readHelper;
- private final boolean isClusterPrecisionInputTable;
-
- public TimelineMetricClusterAggregator(AGGREGATOR_NAME aggregatorName,
- TimelineMetricMetadataManager metricMetadataManager,
- PhoenixHBaseAccessor hBaseAccessor,
- Configuration metricsConf,
- String checkpointLocation,
- Long sleepIntervalMillis,
- Integer checkpointCutOffMultiplier,
- String hostAggregatorDisabledParam,
- String inputTableName,
- String outputTableName,
- Long nativeTimeRangeDelay,
- MetricCollectorHAController haController) {
- super(aggregatorName, hBaseAccessor, metricsConf, checkpointLocation,
- sleepIntervalMillis, checkpointCutOffMultiplier,
- hostAggregatorDisabledParam, inputTableName, outputTableName,
- nativeTimeRangeDelay, haController);
- isClusterPrecisionInputTable = inputTableName.equals(METRICS_CLUSTER_AGGREGATE_TABLE_NAME);
- readHelper = new TimelineMetricReadHelper(metricMetadataManager, true);
- }
-
- @Override
- protected Condition prepareMetricQueryCondition(long startTime, long endTime) {
- Condition condition = new DefaultCondition(null, null, null, null, startTime,
- endTime, null, null, true);
- condition.setNoLimit();
- condition.setFetchSize(resultsetFetchSize);
- String sqlStr = String.format(GET_CLUSTER_AGGREGATE_TIME_SQL, tableName);
- // HOST_COUNT vs METRIC_COUNT
- if (isClusterPrecisionInputTable) {
- sqlStr = String.format(GET_CLUSTER_AGGREGATE_SQL, tableName);
- }
-
- condition.setStatement(sqlStr);
- condition.addOrderByColumn("UUID");
- condition.addOrderByColumn("SERVER_TIME");
- return condition;
- }
-
- @Override
- protected void aggregate(ResultSet rs, long startTime, long endTime) throws IOException, SQLException {
- Map<TimelineClusterMetric, MetricHostAggregate> hostAggregateMap = aggregateMetricsFromResultSet(rs, endTime);
-
- LOG.info("Saving " + hostAggregateMap.size() + " metric aggregates.");
- hBaseAccessor.saveClusterAggregateRecordsSecond(hostAggregateMap, outputTableName);
- }
-
- private Map<TimelineClusterMetric, MetricHostAggregate> aggregateMetricsFromResultSet(ResultSet rs, long endTime)
- throws IOException, SQLException {
-
- TimelineClusterMetric existingMetric = null;
- MetricHostAggregate hostAggregate = null;
- Map<TimelineClusterMetric, MetricHostAggregate> hostAggregateMap =
- new HashMap<TimelineClusterMetric, MetricHostAggregate>();
- int perMetricCount = 0;
-
- while (rs.next()) {
- TimelineClusterMetric currentMetric = readHelper.fromResultSet(rs);
-
- MetricClusterAggregate currentHostAggregate =
- isClusterPrecisionInputTable ?
- readHelper.getMetricClusterAggregateFromResultSet(rs) :
- readHelper.getMetricClusterTimeAggregateFromResultSet(rs);
-
- if (existingMetric == null) {
- // First row
- existingMetric = currentMetric;
- currentMetric.setTimestamp(endTime);
- hostAggregate = new MetricHostAggregate();
- hostAggregateMap.put(currentMetric, hostAggregate);
- perMetricCount++;
- }
-
- if (existingMetric.equalsExceptTime(currentMetric)) {
- // Recalculate totals with current metric
- updateAggregatesFromHost(hostAggregate, currentHostAggregate);
- perMetricCount++;
- } else {
- // Switched over to a new metric - save new metric
-
- hostAggregate.setSum(hostAggregate.getSum() / perMetricCount);
- hostAggregate.setNumberOfSamples(Math.round((float)hostAggregate.getNumberOfSamples() / (float)perMetricCount));
- perMetricCount = 1;
-
- hostAggregate = new MetricHostAggregate();
- currentMetric.setTimestamp(endTime);
- updateAggregatesFromHost(hostAggregate, currentHostAggregate);
- hostAggregateMap.put(currentMetric, hostAggregate);
- existingMetric = currentMetric;
- }
-
- }
-
- return hostAggregateMap;
- }
-
- private void updateAggregatesFromHost(MetricHostAggregate agg, MetricClusterAggregate currentClusterAggregate) {
- agg.updateMax(currentClusterAggregate.getMax());
- agg.updateMin(currentClusterAggregate.getMin());
- agg.updateSum(currentClusterAggregate.getSum());
- agg.updateNumberOfSamples(currentClusterAggregate.getNumberOfHosts());
- }
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricClusterAggregatorSecond.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricClusterAggregatorSecond.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricClusterAggregatorSecond.java
deleted file mode 100644
index bbe8f7b..0000000
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricClusterAggregatorSecond.java
+++ /dev/null
@@ -1,263 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.aggregators;
-
-
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.SERVER_SIDE_TIMESIFT_ADJUSTMENT;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.TIMELINE_METRICS_CLUSTER_AGGREGATOR_INTERPOLATION_ENABLED;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.TIMELINE_METRIC_AGGREGATION_SQL_FILTERS;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.aggregators.AggregatorUtils.getTimeSlices;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.aggregators.AggregatorUtils.sliceFromTimelineMetric;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query.PhoenixTransactSQL.GET_METRIC_SQL;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query.PhoenixTransactSQL.METRICS_RECORD_TABLE_NAME;
-
-import java.io.IOException;
-import java.sql.ResultSet;
-import java.sql.SQLException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.commons.lang.StringUtils;
-import org.apache.commons.lang.mutable.MutableInt;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.metrics2.sink.timeline.MetricClusterAggregate;
-import org.apache.hadoop.metrics2.sink.timeline.TimelineMetric;
-import org.apache.hadoop.metrics2.sink.timeline.TimelineMetricMetadata;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.PhoenixHBaseAccessor;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.availability.AggregationTaskRunner.AGGREGATOR_NAME;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.availability.MetricCollectorHAController;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.discovery.TimelineMetricMetadataKey;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.discovery.TimelineMetricMetadataManager;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query.Condition;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query.DefaultCondition;
-
-/**
- * Aggregates a metric across all hosts in the cluster. Reads metrics from
- * the precision table and saves into the aggregate.
- */
-public class TimelineMetricClusterAggregatorSecond extends AbstractTimelineAggregator {
- public Long timeSliceIntervalMillis;
- private TimelineMetricReadHelper timelineMetricReadHelper;
- // Aggregator to perform app-level aggregates for host metrics
- private final TimelineMetricAppAggregator appAggregator;
- // 1 minute client side buffering adjustment
- protected final Long serverTimeShiftAdjustment;
- protected final boolean interpolationEnabled;
- private TimelineMetricMetadataManager metadataManagerInstance;
- private String skipAggrPatternStrings;
- private final static String liveHostsMetricName = "live_hosts";
-
- public TimelineMetricClusterAggregatorSecond(AGGREGATOR_NAME aggregatorName,
- TimelineMetricMetadataManager metadataManager,
- PhoenixHBaseAccessor hBaseAccessor,
- Configuration metricsConf,
- String checkpointLocation,
- Long sleepIntervalMillis,
- Integer checkpointCutOffMultiplier,
- String aggregatorDisabledParam,
- String tableName,
- String outputTableName,
- Long nativeTimeRangeDelay,
- Long timeSliceInterval,
- MetricCollectorHAController haController) {
- super(aggregatorName, hBaseAccessor, metricsConf, checkpointLocation,
- sleepIntervalMillis, checkpointCutOffMultiplier, aggregatorDisabledParam,
- tableName, outputTableName, nativeTimeRangeDelay, haController);
-
- this.metadataManagerInstance = metadataManager;
- appAggregator = new TimelineMetricAppAggregator(metadataManager, metricsConf);
- this.timeSliceIntervalMillis = timeSliceInterval;
- this.serverTimeShiftAdjustment = Long.parseLong(metricsConf.get(SERVER_SIDE_TIMESIFT_ADJUSTMENT, "90000"));
- this.interpolationEnabled = Boolean.parseBoolean(metricsConf.get(TIMELINE_METRICS_CLUSTER_AGGREGATOR_INTERPOLATION_ENABLED, "true"));
- this.skipAggrPatternStrings = metricsConf.get(TIMELINE_METRIC_AGGREGATION_SQL_FILTERS);
- this.timelineMetricReadHelper = new TimelineMetricReadHelper(metadataManager, true);
- }
-
- @Override
- protected void aggregate(ResultSet rs, long startTime, long endTime) throws SQLException, IOException {
- // Account for time shift due to client side buffering by shifting the
- // timestamps with the difference between server time and series start time
- // Also, we do not want to look at the shift time period from the end as well since we can interpolate those points
- // that come earlier than the expected, during the next run.
- List<Long[]> timeSlices = getTimeSlices(startTime - serverTimeShiftAdjustment, endTime - serverTimeShiftAdjustment, timeSliceIntervalMillis);
- // Initialize app aggregates for host metrics
- appAggregator.init();
- Map<TimelineClusterMetric, MetricClusterAggregate> aggregateClusterMetrics =
- aggregateMetricsFromResultSet(rs, timeSlices);
-
- LOG.info("Saving " + aggregateClusterMetrics.size() + " metric aggregates.");
- hBaseAccessor.saveClusterAggregateRecords(aggregateClusterMetrics);
- appAggregator.cleanup();
- }
-
- @Override
- protected Condition prepareMetricQueryCondition(long startTime, long endTime) {
-
- List<String> metricNames = new ArrayList<>();
- boolean metricNamesNotCondition = false;
-
- if (!StringUtils.isEmpty(skipAggrPatternStrings)) {
- LOG.info("Skipping aggregation for metric patterns : " + skipAggrPatternStrings);
- metricNames.addAll(Arrays.asList(skipAggrPatternStrings.split(",")));
- metricNamesNotCondition = true;
- }
-
- Condition condition = new DefaultCondition(metricNames, null, null, null, startTime - serverTimeShiftAdjustment,
- endTime, null, null, true);
- condition.setMetricNamesNotCondition(metricNamesNotCondition);
- condition.setNoLimit();
- condition.setFetchSize(resultsetFetchSize);
- condition.setStatement(String.format(GET_METRIC_SQL,
- METRICS_RECORD_TABLE_NAME));
- // Retaining order of the row-key avoids client side merge sort.
- condition.addOrderByColumn("UUID");
- condition.addOrderByColumn("SERVER_TIME");
- return condition;
- }
-
- Map<TimelineClusterMetric, MetricClusterAggregate> aggregateMetricsFromResultSet(ResultSet rs, List<Long[]> timeSlices)
- throws SQLException, IOException {
- Map<TimelineClusterMetric, MetricClusterAggregate> aggregateClusterMetrics =
- new HashMap<TimelineClusterMetric, MetricClusterAggregate>();
-
- TimelineMetric metric = null;
- Map<String, MutableInt> hostedAppCounter = new HashMap<>();
- if (rs.next()) {
- metric = timelineMetricReadHelper.getTimelineMetricFromResultSet(rs);
-
- // Call slice after all rows for a host are read
- while (rs.next()) {
- TimelineMetric nextMetric = timelineMetricReadHelper.getTimelineMetricFromResultSet(rs);
- // If rows belong to same host combine them before slicing. This
- // avoids issues across rows that belong to same hosts but get
- // counted as coming from different ones.
- if (metric.equalsExceptTime(nextMetric)) {
- metric.addMetricValues(nextMetric.getMetricValues());
- } else {
- // Process the current metric
- int numHosts = processAggregateClusterMetrics(aggregateClusterMetrics, metric, timeSlices);
- if (!hostedAppCounter.containsKey(metric.getAppId())) {
- hostedAppCounter.put(metric.getAppId(), new MutableInt(numHosts));
- } else {
- int currentHostCount = hostedAppCounter.get(metric.getAppId()).intValue();
- if (currentHostCount < numHosts) {
- hostedAppCounter.put(metric.getAppId(), new MutableInt(numHosts));
- }
- }
- metric = nextMetric;
- }
- }
- }
- // Process last metric
- if (metric != null) {
- int numHosts = processAggregateClusterMetrics(aggregateClusterMetrics, metric, timeSlices);
- if (!hostedAppCounter.containsKey(metric.getAppId())) {
- hostedAppCounter.put(metric.getAppId(), new MutableInt(numHosts));
- } else {
- int currentHostCount = hostedAppCounter.get(metric.getAppId()).intValue();
- if (currentHostCount < numHosts) {
- hostedAppCounter.put(metric.getAppId(), new MutableInt(numHosts));
- }
- }
- }
-
- // Add app level aggregates to save
- aggregateClusterMetrics.putAll(appAggregator.getAggregateClusterMetrics());
-
- // Add liveHosts per AppId metrics.
- long timestamp = timeSlices.get(timeSlices.size() - 1)[1];
- processLiveAppCountMetrics(aggregateClusterMetrics, hostedAppCounter, timestamp);
-
- return aggregateClusterMetrics;
- }
-
- /**
- * Slice metric values into interval specified by :
- * timeline.metrics.cluster.aggregator.minute.timeslice.interval
- * Normalize value by averaging them within the interval
- */
- protected int processAggregateClusterMetrics(Map<TimelineClusterMetric, MetricClusterAggregate> aggregateClusterMetrics,
- TimelineMetric metric, List<Long[]> timeSlices) {
- // Create time slices
- TimelineMetricMetadataKey appKey = new TimelineMetricMetadataKey(metric.getMetricName(), metric.getAppId(), metric.getInstanceId());
- TimelineMetricMetadata metricMetadata = metadataManagerInstance.getMetadataCacheValue(appKey);
-
- if (metricMetadata != null && !metricMetadata.isSupportsAggregates()) {
- LOG.debug("Skipping cluster aggregation for " + metric.getMetricName());
- return 0;
- }
-
- Map<TimelineClusterMetric, Double> clusterMetrics = sliceFromTimelineMetric(metric, timeSlices, interpolationEnabled);
-
- return aggregateClusterMetricsFromSlices(clusterMetrics, aggregateClusterMetrics, metric.getHostName());
- }
-
- protected int aggregateClusterMetricsFromSlices(Map<TimelineClusterMetric, Double> clusterMetrics,
- Map<TimelineClusterMetric, MetricClusterAggregate> aggregateClusterMetrics,
- String hostname) {
-
- int numHosts = 0;
- if (clusterMetrics != null && !clusterMetrics.isEmpty()) {
- for (Map.Entry<TimelineClusterMetric, Double> clusterMetricEntry : clusterMetrics.entrySet()) {
-
- TimelineClusterMetric clusterMetric = clusterMetricEntry.getKey();
- Double avgValue = clusterMetricEntry.getValue();
-
- MetricClusterAggregate aggregate = aggregateClusterMetrics.get(clusterMetric);
-
- if (aggregate == null) {
- aggregate = new MetricClusterAggregate(avgValue, 1, null, avgValue, avgValue);
- aggregateClusterMetrics.put(clusterMetric, aggregate);
- } else {
- aggregate.updateSum(avgValue);
- aggregate.updateNumberOfHosts(1);
- aggregate.updateMax(avgValue);
- aggregate.updateMin(avgValue);
- }
-
- numHosts = aggregate.getNumberOfHosts();
- // Update app level aggregates
- appAggregator.processTimelineClusterMetric(clusterMetric, hostname, avgValue);
- }
- }
- return numHosts;
- }
-
- /* Add cluster metric for number of hosts that are hosting an appId */
- protected void processLiveAppCountMetrics(Map<TimelineClusterMetric, MetricClusterAggregate> aggregateClusterMetrics,
- Map<String, MutableInt> appHostsCount, long timestamp) {
-
- for (Map.Entry<String, MutableInt> appHostsEntry : appHostsCount.entrySet()) {
- TimelineClusterMetric timelineClusterMetric = new TimelineClusterMetric(
- liveHostsMetricName, appHostsEntry.getKey(), null, timestamp);
-
- Integer numOfHosts = appHostsEntry.getValue().intValue();
-
- MetricClusterAggregate metricClusterAggregate = new MetricClusterAggregate(
- (double) numOfHosts, 1, null, (double) numOfHosts, (double) numOfHosts);
-
- metadataManagerInstance.getUuid(timelineClusterMetric);
-
- aggregateClusterMetrics.put(timelineClusterMetric, metricClusterAggregate);
- }
-
- }
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricClusterAggregatorSecondWithCacheSource.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricClusterAggregatorSecondWithCacheSource.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricClusterAggregatorSecondWithCacheSource.java
deleted file mode 100644
index dc31086..0000000
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricClusterAggregatorSecondWithCacheSource.java
+++ /dev/null
@@ -1,104 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.aggregators;
-
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.aggregators.AggregatorUtils.getTimeSlices;
-
-import java.util.Date;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.commons.lang.mutable.MutableInt;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.metrics2.sink.timeline.MetricClusterAggregate;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.PhoenixHBaseAccessor;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricDistributedCache;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.availability.AggregationTaskRunner;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.availability.MetricCollectorHAController;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.discovery.TimelineMetricMetadataManager;
-
-public class TimelineMetricClusterAggregatorSecondWithCacheSource extends TimelineMetricClusterAggregatorSecond {
- private TimelineMetricDistributedCache distributedCache;
- public TimelineMetricClusterAggregatorSecondWithCacheSource(AggregationTaskRunner.AGGREGATOR_NAME metricAggregateSecond, TimelineMetricMetadataManager metricMetadataManager, PhoenixHBaseAccessor hBaseAccessor, Configuration metricsConf, String checkpointLocation, long sleepIntervalMillis, int checkpointCutOffMultiplier, String aggregatorDisabledParam, String inputTableName, String outputTableName,
- Long nativeTimeRangeDelay,
- Long timeSliceInterval,
- MetricCollectorHAController haController, TimelineMetricDistributedCache distributedCache) {
- super(metricAggregateSecond, metricMetadataManager, hBaseAccessor, metricsConf, checkpointLocation, sleepIntervalMillis, checkpointCutOffMultiplier, aggregatorDisabledParam, inputTableName, outputTableName, nativeTimeRangeDelay, timeSliceInterval, haController);
- this.distributedCache = distributedCache;
- }
-
- @Override
- public boolean doWork(long startTime, long endTime) {
- LOG.info("Start aggregation cycle @ " + new Date() + ", " +
- "startTime = " + new Date(startTime) + ", endTime = " + new Date(endTime));
- try {
- Map<String, Double> caheMetrics;
- if (LOG.isDebugEnabled()) {
- caheMetrics = distributedCache.getPointInTimeCacheMetrics();
- LOG.debug("Ignite metrics before eviction : " + caheMetrics);
- }
-
- LOG.info("Trying to evict elements from cache");
- Map<TimelineClusterMetric, MetricClusterAggregate> metricsFromCache = distributedCache.evictMetricAggregates(startTime - serverTimeShiftAdjustment, endTime - serverTimeShiftAdjustment);
- LOG.info(String.format("Evicted %s elements from cache.", metricsFromCache.size()));
-
- if (LOG.isDebugEnabled()) {
- caheMetrics = distributedCache.getPointInTimeCacheMetrics();
- LOG.debug("Ignite metrics after eviction : " + caheMetrics);
- }
-
- List<Long[]> timeSlices = getTimeSlices(startTime - serverTimeShiftAdjustment, endTime - serverTimeShiftAdjustment, timeSliceIntervalMillis);
- Map<TimelineClusterMetric, MetricClusterAggregate> result = aggregateMetricsFromMetricClusterAggregates(metricsFromCache, timeSlices);
-
- LOG.info("Saving " + result.size() + " metric aggregates.");
- hBaseAccessor.saveClusterAggregateRecords(result);
- LOG.info("End aggregation cycle @ " + new Date());
- return true;
- } catch (Exception e) {
- LOG.error("Exception during aggregation. ", e);
- return false;
- }
- }
-
- //Slices in cache could be different from aggregate slices, so need to recalculate. Counts hosted apps
- Map<TimelineClusterMetric, MetricClusterAggregate> aggregateMetricsFromMetricClusterAggregates(Map<TimelineClusterMetric, MetricClusterAggregate> metricsFromCache, List<Long[]> timeSlices) {
- //TODO add basic interpolation
- //TODO investigate if needed, maybe add config to disable/enable
- //count hosted apps
- Map<String, MutableInt> hostedAppCounter = new HashMap<>();
- for (Map.Entry<TimelineClusterMetric, MetricClusterAggregate> clusterMetricAggregateEntry : metricsFromCache.entrySet()) {
- int numHosts = clusterMetricAggregateEntry.getValue().getNumberOfHosts();
- String appId = clusterMetricAggregateEntry.getKey().getAppId();
- if (!hostedAppCounter.containsKey(appId)) {
- hostedAppCounter.put(appId, new MutableInt(numHosts));
- } else {
- int currentHostCount = hostedAppCounter.get(appId).intValue();
- if (currentHostCount < numHosts) {
- hostedAppCounter.put(appId, new MutableInt(numHosts));
- }
- }
- }
-
- // Add liveHosts per AppId metrics.
- processLiveAppCountMetrics(metricsFromCache, hostedAppCounter, timeSlices.get(timeSlices.size() - 1)[1]);
-
- return metricsFromCache;
- }
-
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricFilteringHostAggregator.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricFilteringHostAggregator.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricFilteringHostAggregator.java
deleted file mode 100644
index a75d2c4..0000000
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricFilteringHostAggregator.java
+++ /dev/null
@@ -1,94 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.aggregators;
-
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query.PhoenixTransactSQL.GET_METRIC_AGGREGATE_ONLY_SQL;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.PhoenixHBaseAccessor;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.availability.AggregationTaskRunner;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.availability.MetricCollectorHAController;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.discovery.TimelineMetricMetadataManager;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query.Condition;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query.DefaultCondition;
-
-public class TimelineMetricFilteringHostAggregator extends TimelineMetricHostAggregator {
- private static final Log LOG = LogFactory.getLog(TimelineMetricFilteringHostAggregator.class);
- private TimelineMetricMetadataManager metricMetadataManager;
- private ConcurrentHashMap<String, Long> postedAggregatedMap;
-
- public TimelineMetricFilteringHostAggregator(AggregationTaskRunner.AGGREGATOR_NAME aggregatorName,
- TimelineMetricMetadataManager metricMetadataManager,
- PhoenixHBaseAccessor hBaseAccessor,
- Configuration metricsConf,
- String checkpointLocation,
- Long sleepIntervalMillis,
- Integer checkpointCutOffMultiplier,
- String hostAggregatorDisabledParam,
- String tableName,
- String outputTableName,
- Long nativeTimeRangeDelay,
- MetricCollectorHAController haController,
- ConcurrentHashMap<String, Long> postedAggregatedMap) {
- super(aggregatorName, metricMetadataManager,
- hBaseAccessor, metricsConf,
- checkpointLocation,
- sleepIntervalMillis,
- checkpointCutOffMultiplier,
- hostAggregatorDisabledParam,
- tableName,
- outputTableName,
- nativeTimeRangeDelay,
- haController);
- this.metricMetadataManager = metricMetadataManager;
- this.postedAggregatedMap = postedAggregatedMap;
- }
-
- @Override
- protected Condition prepareMetricQueryCondition(long startTime, long endTime) {
- List<String> aggregatedHostnames = new ArrayList<>();
- for (Map.Entry<String, Long> entry : postedAggregatedMap.entrySet()) {
- if (entry.getValue() > startTime && entry.getValue() <= endTime) {
- aggregatedHostnames.add(entry.getKey());
- }
- }
- List<String> notAggregatedHostnames = metricMetadataManager.getNotLikeHostnames(aggregatedHostnames);
- if (LOG.isDebugEnabled()) {
- LOG.debug("Already aggregated hostnames based on postedAggregatedMap : " + aggregatedHostnames);
- LOG.debug("Hostnames that will be aggregated : " + notAggregatedHostnames);
- }
- List<byte[]> uuids = metricMetadataManager.getUuids(new ArrayList<String>(), notAggregatedHostnames, "", "");
-
- Condition condition = new DefaultCondition(uuids, null, null, null, null, startTime,
- endTime, null, null, true);
- condition.setNoLimit();
- condition.setFetchSize(resultsetFetchSize);
- condition.setStatement(String.format(GET_METRIC_AGGREGATE_ONLY_SQL, tableName));
- // Retaining order of the row-key avoids client side merge sort.
- condition.addOrderByColumn("UUID");
- condition.addOrderByColumn("SERVER_TIME");
- return condition;
- }
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricHostAggregator.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricHostAggregator.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricHostAggregator.java
deleted file mode 100644
index 6a11599..0000000
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricHostAggregator.java
+++ /dev/null
@@ -1,121 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.aggregators;
-
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query.PhoenixTransactSQL.GET_METRIC_AGGREGATE_ONLY_SQL;
-
-import java.io.IOException;
-import java.sql.ResultSet;
-import java.sql.SQLException;
-import java.util.HashMap;
-import java.util.Map;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.metrics2.sink.timeline.MetricHostAggregate;
-import org.apache.hadoop.metrics2.sink.timeline.TimelineMetric;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.PhoenixHBaseAccessor;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.availability.AggregationTaskRunner.AGGREGATOR_NAME;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.availability.MetricCollectorHAController;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.discovery.TimelineMetricMetadataManager;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query.Condition;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query.DefaultCondition;
-
-public class TimelineMetricHostAggregator extends AbstractTimelineAggregator {
- private static final Log LOG = LogFactory.getLog(TimelineMetricHostAggregator.class);
- TimelineMetricReadHelper readHelper;
-
- public TimelineMetricHostAggregator(AGGREGATOR_NAME aggregatorName,
- TimelineMetricMetadataManager metricMetadataManager,
- PhoenixHBaseAccessor hBaseAccessor,
- Configuration metricsConf,
- String checkpointLocation,
- Long sleepIntervalMillis,
- Integer checkpointCutOffMultiplier,
- String hostAggregatorDisabledParam,
- String tableName,
- String outputTableName,
- Long nativeTimeRangeDelay,
- MetricCollectorHAController haController) {
- super(aggregatorName, hBaseAccessor, metricsConf, checkpointLocation,
- sleepIntervalMillis, checkpointCutOffMultiplier, hostAggregatorDisabledParam,
- tableName, outputTableName, nativeTimeRangeDelay, haController);
- readHelper = new TimelineMetricReadHelper(metricMetadataManager, false);
- }
-
- @Override
- protected void aggregate(ResultSet rs, long startTime, long endTime) throws IOException, SQLException {
-
- Map<TimelineMetric, MetricHostAggregate> hostAggregateMap = aggregateMetricsFromResultSet(rs, endTime);
-
- LOG.info("Saving " + hostAggregateMap.size() + " metric aggregates.");
- hBaseAccessor.saveHostAggregateRecords(hostAggregateMap, outputTableName);
- }
-
- @Override
- protected Condition prepareMetricQueryCondition(long startTime, long endTime) {
- Condition condition = new DefaultCondition(null, null, null, null, startTime,
- endTime, null, null, true);
- condition.setNoLimit();
- condition.setFetchSize(resultsetFetchSize);
- condition.setStatement(String.format(GET_METRIC_AGGREGATE_ONLY_SQL, tableName));
- // Retaining order of the row-key avoids client side merge sort.
- condition.addOrderByColumn("UUID");
- condition.addOrderByColumn("SERVER_TIME");
- return condition;
- }
-
- private Map<TimelineMetric, MetricHostAggregate> aggregateMetricsFromResultSet(ResultSet rs, long endTime)
- throws IOException, SQLException {
- TimelineMetric existingMetric = null;
- MetricHostAggregate hostAggregate = null;
- Map<TimelineMetric, MetricHostAggregate> hostAggregateMap = new HashMap<TimelineMetric, MetricHostAggregate>();
-
-
- while (rs.next()) {
- TimelineMetric currentMetric =
- readHelper.getTimelineMetricKeyFromResultSet(rs);
- MetricHostAggregate currentHostAggregate =
- readHelper.getMetricHostAggregateFromResultSet(rs);
-
- if (existingMetric == null) {
- // First row
- existingMetric = currentMetric;
- currentMetric.setStartTime(endTime);
- hostAggregate = new MetricHostAggregate();
- hostAggregateMap.put(currentMetric, hostAggregate);
- }
-
- if (existingMetric.equalsExceptTime(currentMetric)) {
- // Recalculate totals with current metric
- hostAggregate.updateAggregates(currentHostAggregate);
- } else {
- // Switched over to a new metric - save existing - create new aggregate
- currentMetric.setStartTime(endTime);
- hostAggregate = new MetricHostAggregate();
- hostAggregate.updateAggregates(currentHostAggregate);
- hostAggregateMap.put(currentMetric, hostAggregate);
- existingMetric = currentMetric;
- }
- }
- return hostAggregateMap;
- }
-
-
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricReadHelper.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricReadHelper.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricReadHelper.java
deleted file mode 100644
index 5d31b51..0000000
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricReadHelper.java
+++ /dev/null
@@ -1,170 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.aggregators;
-
-
-import java.io.IOException;
-import java.sql.ResultSet;
-import java.sql.SQLException;
-import java.util.TreeMap;
-
-import org.apache.hadoop.metrics2.sink.timeline.MetricClusterAggregate;
-import org.apache.hadoop.metrics2.sink.timeline.MetricHostAggregate;
-import org.apache.hadoop.metrics2.sink.timeline.SingleValuedTimelineMetric;
-import org.apache.hadoop.metrics2.sink.timeline.TimelineMetric;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.PhoenixHBaseAccessor;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.discovery.TimelineMetricMetadataManager;
-
-public class TimelineMetricReadHelper {
-
- private boolean ignoreInstance = false;
- private TimelineMetricMetadataManager metadataManagerInstance = null;
-
- public TimelineMetricReadHelper() {}
-
- public TimelineMetricReadHelper(boolean ignoreInstance) {
- this.ignoreInstance = ignoreInstance;
- }
-
- public TimelineMetricReadHelper(TimelineMetricMetadataManager timelineMetricMetadataManager) {
- this.metadataManagerInstance = timelineMetricMetadataManager;
- }
-
- public TimelineMetricReadHelper(TimelineMetricMetadataManager timelineMetricMetadataManager, boolean ignoreInstance) {
- this.metadataManagerInstance = timelineMetricMetadataManager;
- this.ignoreInstance = ignoreInstance;
- }
-
- public TimelineMetric getTimelineMetricFromResultSet(ResultSet rs)
- throws SQLException, IOException {
- TimelineMetric metric = getTimelineMetricCommonsFromResultSet(rs);
- TreeMap<Long, Double> sortedByTimeMetrics = PhoenixHBaseAccessor.readMetricFromJSON(rs.getString("METRICS"));
- metric.setMetricValues(sortedByTimeMetrics);
- return metric;
- }
-
- public SingleValuedTimelineMetric getAggregatedTimelineMetricFromResultSet(ResultSet rs,
- Function f) throws SQLException, IOException {
-
- byte[] uuid = rs.getBytes("UUID");
- TimelineMetric timelineMetric = metadataManagerInstance.getMetricFromUuid(uuid);
- Function function = (f != null) ? f : Function.DEFAULT_VALUE_FUNCTION;
- SingleValuedTimelineMetric metric = new SingleValuedTimelineMetric(
- timelineMetric.getMetricName() + function.getSuffix(),
- timelineMetric.getAppId(),
- timelineMetric.getInstanceId(),
- timelineMetric.getHostName(),
- rs.getLong("SERVER_TIME")
- );
-
- double value;
- switch(function.getReadFunction()){
- case AVG:
- value = rs.getDouble("METRIC_SUM") / rs.getInt("METRIC_COUNT");
- break;
- case MIN:
- value = rs.getDouble("METRIC_MIN");
- break;
- case MAX:
- value = rs.getDouble("METRIC_MAX");
- break;
- case SUM:
- value = rs.getDouble("METRIC_SUM") / rs.getInt("METRIC_COUNT");
- break;
- default:
- value = rs.getDouble("METRIC_SUM") / rs.getInt("METRIC_COUNT");
- break;
- }
-
- metric.setSingleTimeseriesValue(rs.getLong("SERVER_TIME"), value);
-
- return metric;
- }
-
- /**
- * Returns common part of timeline metrics record without the values.
- */
- public TimelineMetric getTimelineMetricCommonsFromResultSet(ResultSet rs)
- throws SQLException {
-
- byte[] uuid = rs.getBytes("UUID");
- TimelineMetric metric = metadataManagerInstance.getMetricFromUuid(uuid);
- if (ignoreInstance) {
- metric.setInstanceId(null);
- }
- metric.setStartTime(rs.getLong("SERVER_TIME"));
- return metric;
- }
-
- public MetricClusterAggregate getMetricClusterAggregateFromResultSet(ResultSet rs)
- throws SQLException {
- MetricClusterAggregate agg = new MetricClusterAggregate();
- agg.setSum(rs.getDouble("METRIC_SUM"));
- agg.setMax(rs.getDouble("METRIC_MAX"));
- agg.setMin(rs.getDouble("METRIC_MIN"));
- agg.setNumberOfHosts(rs.getInt("HOSTS_COUNT"));
-
- agg.setDeviation(0.0);
-
- return agg;
- }
-
- public MetricClusterAggregate getMetricClusterTimeAggregateFromResultSet(ResultSet rs)
- throws SQLException {
- MetricClusterAggregate agg = new MetricClusterAggregate();
- agg.setSum(rs.getDouble("METRIC_SUM"));
- agg.setMax(rs.getDouble("METRIC_MAX"));
- agg.setMin(rs.getDouble("METRIC_MIN"));
- agg.setNumberOfHosts(rs.getInt("METRIC_COUNT"));
-
- agg.setDeviation(0.0);
-
- return agg;
- }
-
- public TimelineClusterMetric fromResultSet(ResultSet rs) throws SQLException {
-
- byte[] uuid = rs.getBytes("UUID");
- TimelineMetric timelineMetric = metadataManagerInstance.getMetricFromUuid(uuid);
-
- return new TimelineClusterMetric(
- timelineMetric.getMetricName(),
- timelineMetric.getAppId(),
- ignoreInstance ? null : timelineMetric.getInstanceId(),
- rs.getLong("SERVER_TIME"));
- }
-
- public MetricHostAggregate getMetricHostAggregateFromResultSet(ResultSet rs)
- throws SQLException {
- MetricHostAggregate metricHostAggregate = new MetricHostAggregate();
- metricHostAggregate.setSum(rs.getDouble("METRIC_SUM"));
- metricHostAggregate.setMax(rs.getDouble("METRIC_MAX"));
- metricHostAggregate.setMin(rs.getDouble("METRIC_MIN"));
- metricHostAggregate.setNumberOfSamples(rs.getLong("METRIC_COUNT"));
-
- metricHostAggregate.setDeviation(0.0);
- return metricHostAggregate;
- }
-
- public TimelineMetric getTimelineMetricKeyFromResultSet(ResultSet rs)
- throws SQLException, IOException {
- byte[] uuid = rs.getBytes("UUID");
- return metadataManagerInstance.getMetricFromUuid(uuid);
- }
-}
-
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TopNDownSampler.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TopNDownSampler.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TopNDownSampler.java
deleted file mode 100644
index d55d026..0000000
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TopNDownSampler.java
+++ /dev/null
@@ -1,112 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.aggregators;
-
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query.PhoenixTransactSQL.TOPN_DOWNSAMPLER_CLUSTER_METRIC_SELECT_SQL;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query.PhoenixTransactSQL.TOPN_DOWNSAMPLER_HOST_METRIC_SELECT_SQL;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.commons.lang3.StringUtils;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.metrics2.sink.timeline.TopNConfig;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query.TopNCondition;
-
-public class TopNDownSampler implements CustomDownSampler {
-
- private TopNConfig topNConfig;
- private static final Log LOG = LogFactory.getLog(TopNDownSampler.class);
- protected String metricPatterns;
-
- public static TopNDownSampler fromConfig(Map<String, String> conf) {
- String metricPatterns = conf.get(DownSamplerUtils.downSamplerConfigPrefix + "topn." +
- DownSamplerUtils.downSamplerMetricPatternsConfig);
-
- String topNString = conf.get(DownSamplerUtils.downSamplerConfigPrefix + "topn.value");
- Integer topNValue = topNString != null ? Integer.valueOf(topNString) : 10;
- String topNFunction = conf.get(DownSamplerUtils.downSamplerConfigPrefix + "topn.function");
-
- return new TopNDownSampler(new TopNConfig(topNValue, topNFunction, false), metricPatterns);
- }
-
- public TopNDownSampler(TopNConfig topNConfig, String metricPatterns) {
- this.topNConfig = topNConfig;
- this.metricPatterns = metricPatterns;
- }
-
- @Override
- public boolean validateConfigs() {
- if (topNConfig == null) {
- return false;
- }
-
- if (topNConfig.getTopN() <= 0) {
- return false;
- }
-
- if (StringUtils.isEmpty(metricPatterns)) {
- return false;
- }
-
- return true;
- }
-
- /**
- * Prepare downsampling SELECT statement(s) used to determine the data to be written into the Aggregate table.
- * @param startTime
- * @param endTime
- * @param tableName
- * @return
- */
- @Override
- public List<String> prepareDownSamplingStatement(Long startTime, Long endTime, String tableName) {
- List<String> stmts = new ArrayList<>();
-
- Function.ReadFunction readFunction = Function.ReadFunction.getFunction(topNConfig.getTopNFunction());
- Function function = new Function(readFunction, null);
- String columnSelect = TopNCondition.getColumnSelect(function);
-
- List<String> metricPatternList = Arrays.asList(metricPatterns.split(","));
-
- for (String metricPattern : metricPatternList) {
- String metricPatternClause = "'" + metricPattern + "'";
- //TODO : Need a better way to find out what kind of aggregation the current one is.
- if (tableName.contains("RECORD")) {
- stmts.add(String.format(TOPN_DOWNSAMPLER_HOST_METRIC_SELECT_SQL,
- endTime, columnSelect, columnSelect, columnSelect, tableName, metricPatternClause,
- startTime, endTime, columnSelect, topNConfig.getTopN()));
- } else {
- stmts.add(String.format(TOPN_DOWNSAMPLER_CLUSTER_METRIC_SELECT_SQL,
- endTime, columnSelect, columnSelect, columnSelect, tableName, metricPatternClause,
- startTime, endTime, columnSelect, topNConfig.getTopN()));
- }
- }
-
- if (LOG.isDebugEnabled()) {
- LOG.debug("DownSampling Stmt: " + stmts.toString());
- }
-
- return stmts;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/v2/TimelineMetricClusterAggregator.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/v2/TimelineMetricClusterAggregator.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/v2/TimelineMetricClusterAggregator.java
deleted file mode 100644
index 06552a6..0000000
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/v2/TimelineMetricClusterAggregator.java
+++ /dev/null
@@ -1,93 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p/>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p/>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.aggregators.v2;
-
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query.PhoenixTransactSQL.GET_AGGREGATED_APP_METRIC_GROUPBY_SQL;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query.PhoenixTransactSQL.METRICS_CLUSTER_AGGREGATE_TABLE_NAME;
-
-import java.io.IOException;
-import java.sql.ResultSet;
-import java.sql.SQLException;
-import java.util.Date;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.PhoenixHBaseAccessor;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.aggregators.AbstractTimelineAggregator;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.availability.AggregationTaskRunner.AGGREGATOR_NAME;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.availability.MetricCollectorHAController;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query.Condition;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query.EmptyCondition;
-
-public class TimelineMetricClusterAggregator extends AbstractTimelineAggregator {
- private final String aggregateColumnName;
-
- public TimelineMetricClusterAggregator(AGGREGATOR_NAME aggregatorName,
- PhoenixHBaseAccessor hBaseAccessor,
- Configuration metricsConf,
- String checkpointLocation,
- Long sleepIntervalMillis,
- Integer checkpointCutOffMultiplier,
- String hostAggregatorDisabledParam,
- String inputTableName,
- String outputTableName,
- Long nativeTimeRangeDelay,
- MetricCollectorHAController haController) {
- super(aggregatorName, hBaseAccessor, metricsConf, checkpointLocation,
- sleepIntervalMillis, checkpointCutOffMultiplier,
- hostAggregatorDisabledParam, inputTableName, outputTableName,
- nativeTimeRangeDelay, haController);
-
- if (inputTableName.equals(METRICS_CLUSTER_AGGREGATE_TABLE_NAME)) {
- aggregateColumnName = "HOSTS_COUNT";
- } else {
- aggregateColumnName = "METRIC_COUNT";
- }
- }
-
- @Override
- protected Condition prepareMetricQueryCondition(long startTime, long endTime) {
- EmptyCondition condition = new EmptyCondition();
- condition.setDoUpdate(true);
-
- /*
- UPSERT INTO METRIC_AGGREGATE_HOURLY (METRIC_NAME, APP_ID, INSTANCE_ID,
- SERVER_TIME, UNITS, METRIC_SUM, METRIC_COUNT, METRIC_MAX, METRIC_MIN)
- SELECT METRIC_NAME, APP_ID, INSTANCE_ID, MAX(SERVER_TIME), UNITS,
- SUM(METRIC_SUM), SUM(HOSTS_COUNT), MAX(METRIC_MAX), MIN(METRIC_MIN)
- FROM METRIC_AGGREGATE WHERE SERVER_TIME >= 1441155600000 AND
- SERVER_TIME < 1441159200000 GROUP BY METRIC_NAME, APP_ID, INSTANCE_ID, UNITS;
- */
-
- condition.setStatement(String.format(GET_AGGREGATED_APP_METRIC_GROUPBY_SQL,
- outputTableName, endTime, aggregateColumnName, tableName,
- getDownsampledMetricSkipClause(), startTime, endTime));
-
- if (LOG.isDebugEnabled()) {
- LOG.debug("Condition: " + condition.toString());
- }
-
- return condition;
- }
-
- @Override
- protected void aggregate(ResultSet rs, long startTime, long endTime) throws IOException, SQLException {
- LOG.info("Aggregated cluster metrics for " + outputTableName +
- ", with startTime = " + new Date(startTime) +
- ", endTime = " + new Date(endTime));
- }
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/v2/TimelineMetricFilteringHostAggregator.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/v2/TimelineMetricFilteringHostAggregator.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/v2/TimelineMetricFilteringHostAggregator.java
deleted file mode 100644
index a15ab2e..0000000
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/v2/TimelineMetricFilteringHostAggregator.java
+++ /dev/null
@@ -1,119 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.aggregators.v2;
-
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query.PhoenixTransactSQL.GET_AGGREGATED_HOST_METRIC_GROUPBY_SQL;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
-
-import org.apache.commons.collections.CollectionUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.PhoenixHBaseAccessor;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.availability.AggregationTaskRunner;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.availability.MetricCollectorHAController;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.discovery.TimelineMetricMetadataManager;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query.Condition;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query.EmptyCondition;
-
-public class TimelineMetricFilteringHostAggregator extends TimelineMetricHostAggregator {
- private TimelineMetricMetadataManager metricMetadataManager;
- private ConcurrentHashMap<String, Long> postedAggregatedMap;
-
- public TimelineMetricFilteringHostAggregator(AggregationTaskRunner.AGGREGATOR_NAME aggregatorName,
- TimelineMetricMetadataManager metricMetadataManager,
- PhoenixHBaseAccessor hBaseAccessor,
- Configuration metricsConf,
- String checkpointLocation,
- Long sleepIntervalMillis,
- Integer checkpointCutOffMultiplier,
- String hostAggregatorDisabledParam,
- String tableName,
- String outputTableName,
- Long nativeTimeRangeDelay,
- MetricCollectorHAController haController,
- ConcurrentHashMap<String, Long> postedAggregatedMap) {
- super(aggregatorName,
- hBaseAccessor, metricsConf,
- checkpointLocation,
- sleepIntervalMillis,
- checkpointCutOffMultiplier,
- hostAggregatorDisabledParam,
- tableName,
- outputTableName,
- nativeTimeRangeDelay,
- haController);
- this.metricMetadataManager = metricMetadataManager;
- this.postedAggregatedMap = postedAggregatedMap;
- }
-
- @Override
- protected Condition prepareMetricQueryCondition(long startTime, long endTime) {
- List<String> aggregatedHostnames = new ArrayList<>();
- for (Map.Entry<String, Long> entry : postedAggregatedMap.entrySet()) {
- if (entry.getValue() > startTime && entry.getValue() <= endTime) {
- aggregatedHostnames.add(entry.getKey());
- }
- }
- List<String> notAggregatedHostnames = metricMetadataManager.getNotLikeHostnames(aggregatedHostnames);
- if (LOG.isDebugEnabled()) {
- LOG.debug("Already aggregated hostnames based on postedAggregatedMap : " + aggregatedHostnames);
- LOG.debug("Hostnames that will be aggregated : " + notAggregatedHostnames);
- }
- List<byte[]> uuids = metricMetadataManager.getUuids(new ArrayList<String>(), notAggregatedHostnames, "", "");
-
- EmptyCondition condition = new EmptyCondition();
- condition.setDoUpdate(true);
-
- condition.setStatement(String.format(GET_AGGREGATED_HOST_METRIC_GROUPBY_SQL,
- outputTableName, endTime, tableName,
- getDownsampledMetricSkipClause() + getIncludedUuidsClause(uuids), startTime, endTime));
-
- if (LOG.isDebugEnabled()) {
- LOG.debug("Condition: " + condition.toString());
- }
-
- return condition;
- }
-
- private String getIncludedUuidsClause(List<byte[]> uuids) {
- StringBuilder sb = new StringBuilder();
- sb.append("(");
-
- //LIKE clause
- // (UUID LIKE ? OR UUID LIKE ?) AND
- if (CollectionUtils.isNotEmpty(uuids)) {
- for (int i = 0; i < uuids.size(); i++) {
- sb.append("UUID");
- sb.append(" LIKE ");
- sb.append("'%");
- sb.append(new String(uuids.get(i)));
- sb.append("'");
-
- if (i == uuids.size() - 1) {
- sb.append(") AND ");
- } else {
- sb.append(" OR ");
- }
- }
- }
- return sb.toString();
- }
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/v2/TimelineMetricHostAggregator.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/v2/TimelineMetricHostAggregator.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/v2/TimelineMetricHostAggregator.java
deleted file mode 100644
index 3eb2be3..0000000
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/v2/TimelineMetricHostAggregator.java
+++ /dev/null
@@ -1,76 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.aggregators.v2;
-
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query.PhoenixTransactSQL.GET_AGGREGATED_HOST_METRIC_GROUPBY_SQL;
-
-import java.io.IOException;
-import java.sql.ResultSet;
-import java.sql.SQLException;
-import java.util.Date;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.PhoenixHBaseAccessor;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.aggregators.AbstractTimelineAggregator;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.availability.AggregationTaskRunner.AGGREGATOR_NAME;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.availability.MetricCollectorHAController;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query.Condition;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query.EmptyCondition;
-
-public class TimelineMetricHostAggregator extends AbstractTimelineAggregator {
-
- public TimelineMetricHostAggregator(AGGREGATOR_NAME aggregatorName,
- PhoenixHBaseAccessor hBaseAccessor,
- Configuration metricsConf,
- String checkpointLocation,
- Long sleepIntervalMillis,
- Integer checkpointCutOffMultiplier,
- String hostAggregatorDisabledParam,
- String tableName,
- String outputTableName,
- Long nativeTimeRangeDelay,
- MetricCollectorHAController haController) {
- super(aggregatorName, hBaseAccessor, metricsConf, checkpointLocation,
- sleepIntervalMillis, checkpointCutOffMultiplier, hostAggregatorDisabledParam,
- tableName, outputTableName, nativeTimeRangeDelay, haController);
- }
-
- @Override
- protected void aggregate(ResultSet rs, long startTime, long endTime) throws IOException, SQLException {
-
- LOG.info("Aggregated host metrics for " + outputTableName +
- ", with startTime = " + new Date(startTime) +
- ", endTime = " + new Date(endTime));
- }
-
- @Override
- protected Condition prepareMetricQueryCondition(long startTime, long endTime) {
- EmptyCondition condition = new EmptyCondition();
- condition.setDoUpdate(true);
-
- condition.setStatement(String.format(GET_AGGREGATED_HOST_METRIC_GROUPBY_SQL,
- outputTableName, endTime, tableName,
- getDownsampledMetricSkipClause(), startTime, endTime));
-
- if (LOG.isDebugEnabled()) {
- LOG.debug("Condition: " + condition.toString());
- }
-
- return condition;
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/availability/AggregationTaskRunner.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/availability/AggregationTaskRunner.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/availability/AggregationTaskRunner.java
deleted file mode 100644
index fef9dc9..0000000
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/availability/AggregationTaskRunner.java
+++ /dev/null
@@ -1,145 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p/>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p/>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.availability;
-
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.aggregators.TimelineMetricAggregator.AGGREGATOR_TYPE.CLUSTER;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.aggregators.TimelineMetricAggregator.AGGREGATOR_TYPE.HOST;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.availability.AggregationTaskRunner.AGGREGATOR_NAME.METRIC_AGGREGATE_DAILY;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.availability.AggregationTaskRunner.AGGREGATOR_NAME.METRIC_AGGREGATE_HOURLY;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.availability.AggregationTaskRunner.AGGREGATOR_NAME.METRIC_AGGREGATE_MINUTE;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.availability.AggregationTaskRunner.AGGREGATOR_NAME.METRIC_AGGREGATE_SECOND;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.availability.AggregationTaskRunner.AGGREGATOR_NAME.METRIC_RECORD_DAILY;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.availability.AggregationTaskRunner.AGGREGATOR_NAME.METRIC_RECORD_HOURLY;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.availability.AggregationTaskRunner.AGGREGATOR_NAME.METRIC_RECORD_MINUTE;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.availability.MetricCollectorHAController.DEFAULT_STATE_MODEL;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.availability.MetricCollectorHAController.METRIC_AGGREGATORS;
-
-import java.util.HashMap;
-import java.util.Map;
-import java.util.concurrent.atomic.AtomicBoolean;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.aggregators.TimelineMetricAggregator.AGGREGATOR_TYPE;
-import org.apache.helix.HelixManager;
-import org.apache.helix.HelixManagerFactory;
-import org.apache.helix.InstanceType;
-import org.apache.helix.participant.StateMachineEngine;
-
-public class AggregationTaskRunner {
- private final String instanceName;
- private final String zkAddress;
- private final String clusterName;
- private HelixManager manager;
- private static final Log LOG = LogFactory.getLog(AggregationTaskRunner.class);
- private CheckpointManager checkpointManager;
- // Map partition name to an aggregator dimension
- static final Map<String, AGGREGATOR_TYPE> PARTITION_AGGREGATION_TYPES = new HashMap<>();
- // Ownership flags to be set by the State transitions
- private final AtomicBoolean performsClusterAggregation = new AtomicBoolean(false);
- private final AtomicBoolean performsHostAggregation = new AtomicBoolean(false);
-
- public enum AGGREGATOR_NAME {
- METRIC_RECORD_MINUTE,
- METRIC_RECORD_HOURLY,
- METRIC_RECORD_DAILY,
- METRIC_AGGREGATE_SECOND,
- METRIC_AGGREGATE_MINUTE,
- METRIC_AGGREGATE_HOURLY,
- METRIC_AGGREGATE_DAILY,
- }
-
- public static final Map<AGGREGATOR_NAME, String> ACTUAL_AGGREGATOR_NAMES = new HashMap<>();
-
- static {
- ACTUAL_AGGREGATOR_NAMES.put(METRIC_RECORD_MINUTE, "TimelineMetricHostAggregatorMinute");
- ACTUAL_AGGREGATOR_NAMES.put(METRIC_RECORD_HOURLY, "TimelineMetricHostAggregatorHourly");
- ACTUAL_AGGREGATOR_NAMES.put(METRIC_RECORD_DAILY, "TimelineMetricHostAggregatorDaily");
- ACTUAL_AGGREGATOR_NAMES.put(METRIC_AGGREGATE_SECOND, "TimelineClusterAggregatorSecond");
- ACTUAL_AGGREGATOR_NAMES.put(METRIC_AGGREGATE_MINUTE, "TimelineClusterAggregatorMinute");
- ACTUAL_AGGREGATOR_NAMES.put(METRIC_AGGREGATE_HOURLY, "TimelineClusterAggregatorHourly");
- ACTUAL_AGGREGATOR_NAMES.put(METRIC_AGGREGATE_DAILY, "TimelineClusterAggregatorDaily");
-
- // Partition name to task assignment
- PARTITION_AGGREGATION_TYPES.put(METRIC_AGGREGATORS + "_0", CLUSTER);
- PARTITION_AGGREGATION_TYPES.put(METRIC_AGGREGATORS + "_1", HOST);
- }
-
- public AggregationTaskRunner(String instanceName, String zkAddress, String clusterName) {
- this.instanceName = instanceName;
- this.zkAddress = zkAddress;
- this.clusterName = clusterName;
- }
-
- public void initialize() throws Exception {
- manager = HelixManagerFactory.getZKHelixManager(clusterName, instanceName,
- InstanceType.PARTICIPANT, zkAddress);
-
- OnlineOfflineStateModelFactory stateModelFactory =
- new OnlineOfflineStateModelFactory(instanceName, this);
-
- StateMachineEngine stateMach = manager.getStateMachineEngine();
- stateMach.registerStateModelFactory(DEFAULT_STATE_MODEL, stateModelFactory);
- manager.connect();
-
- checkpointManager = new CheckpointManager(manager.getHelixPropertyStore());
- }
-
- public boolean performsClusterAggregation() {
- return performsClusterAggregation.get();
- }
-
- public boolean performsHostAggregation() {
- return performsHostAggregation.get();
- }
-
- public CheckpointManager getCheckpointManager() {
- return checkpointManager;
- }
-
- public void setPartitionAggregationFunction(AGGREGATOR_TYPE type) {
- switch (type) {
- case HOST:
- performsHostAggregation.set(true);
- LOG.info("Set host aggregator function for : " + instanceName);
- break;
- case CLUSTER:
- performsClusterAggregation.set(true);
- LOG.info("Set cluster aggregator function for : " + instanceName);
- }
- }
-
- public void unsetPartitionAggregationFunction(AGGREGATOR_TYPE type) {
- switch (type) {
- case HOST:
- performsHostAggregation.set(false);
- LOG.info("Unset host aggregator function for : " + instanceName);
- break;
- case CLUSTER:
- performsClusterAggregation.set(false);
- LOG.info("Unset cluster aggregator function for : " + instanceName);
- }
- }
-
- /**
- * Disconnect participant before controller shutdown
- */
- void stop() {
- manager.disconnect();
- }
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/availability/CheckpointManager.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/availability/CheckpointManager.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/availability/CheckpointManager.java
deleted file mode 100644
index 3293ead..0000000
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/availability/CheckpointManager.java
+++ /dev/null
@@ -1,98 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p/>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p/>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.availability;
-
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.availability.AggregationTaskRunner.ACTUAL_AGGREGATOR_NAMES;
-
-import org.I0Itec.zkclient.DataUpdater;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.availability.AggregationTaskRunner.AGGREGATOR_NAME;
-import org.apache.helix.AccessOption;
-import org.apache.helix.ZNRecord;
-import org.apache.helix.store.zk.ZkHelixPropertyStore;
-import org.apache.zookeeper.data.Stat;
-
-public class CheckpointManager {
- private final ZkHelixPropertyStore<ZNRecord> propertyStore;
- private static final Log LOG = LogFactory.getLog(CheckpointManager.class);
-
- static final String ZNODE_FIELD = "checkpoint";
- static final String CHECKPOINT_PATH_PREFIX = "CHECKPOINTS";
-
- public CheckpointManager(ZkHelixPropertyStore<ZNRecord> propertyStore) {
- this.propertyStore = propertyStore;
- }
-
- /**
- * Read aggregator checkpoint from zookeeper
- *
- * @return timestamp
- */
- public long readCheckpoint(AGGREGATOR_NAME aggregatorName) {
- String path = getCheckpointZKPath(aggregatorName);
- LOG.debug("Reading checkpoint at " + path);
- Stat stat = new Stat();
- ZNRecord znRecord = propertyStore.get(path, stat, AccessOption.PERSISTENT);
- if (LOG.isTraceEnabled()) {
- LOG.trace("Stat => " + stat);
- }
- long checkpoint = znRecord != null ? znRecord.getLongField(ZNODE_FIELD, -1) : -1;
- LOG.debug("Checkpoint value = " + checkpoint);
- return checkpoint;
- }
-
- /**
- * Write aggregator checkpoint in zookeeper
- *
- * @param value timestamp
- * @return sucsess
- */
- public boolean writeCheckpoint(AGGREGATOR_NAME aggregatorName, long value) {
- String path = getCheckpointZKPath(aggregatorName);
- LOG.debug(String.format("Saving checkpoint at %s with value %s", path, value));
- return propertyStore.update(path, new CheckpointDataUpdater(path, value), AccessOption.PERSISTENT);
- }
-
- static class CheckpointDataUpdater implements DataUpdater<ZNRecord> {
- final String path;
- final Long value;
-
- public CheckpointDataUpdater(String path, Long value) {
- this.path = path;
- this.value = value;
- }
-
- @Override
- public ZNRecord update(ZNRecord currentData) {
- if (currentData == null) {
- currentData = new ZNRecord(path);
- }
- currentData.setLongField(ZNODE_FIELD, value);
- return currentData;
- }
- }
-
- String getCheckpointZKPath(AGGREGATOR_NAME aggregatorName) {
- StringBuilder sb = new StringBuilder("/");
- sb.append(CHECKPOINT_PATH_PREFIX);
- sb.append("/");
- sb.append(ACTUAL_AGGREGATOR_NAMES.get(aggregatorName));
- return sb.toString();
- }
-}
[11/28] ambari git commit: AMBARI-22740 : Fix integration test for
HBase in branch-3.0-ams due to UUID changes. (avijayan)
Posted by av...@apache.org.
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/sink/KafkaSinkProvider.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/sink/KafkaSinkProvider.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/sink/KafkaSinkProvider.java
deleted file mode 100644
index 3b34b55..0000000
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/sink/KafkaSinkProvider.java
+++ /dev/null
@@ -1,118 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.sink;
-
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.KAFKA_ACKS;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.KAFKA_BATCH_SIZE;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.KAFKA_BUFFER_MEM;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.KAFKA_LINGER_MS;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.KAFKA_RETRIES;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.KAFKA_SERVERS;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.KAFKA_SINK_TIMEOUT_SECONDS;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.TIMELINE_METRICS_CACHE_COMMIT_INTERVAL;
-
-import java.util.Collection;
-import java.util.Properties;
-import java.util.concurrent.Future;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.metrics2.sink.timeline.TimelineMetrics;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.source.InternalSourceProvider.SOURCE_NAME;
-import org.apache.kafka.clients.producer.KafkaProducer;
-import org.apache.kafka.clients.producer.Producer;
-import org.apache.kafka.clients.producer.ProducerConfig;
-import org.apache.kafka.clients.producer.ProducerRecord;
-import org.apache.kafka.clients.producer.RecordMetadata;
-
-import com.fasterxml.jackson.databind.JsonNode;
-import com.fasterxml.jackson.databind.ObjectMapper;
-
-/*
- This will be used by the single Metrics committer thread. Hence it is
- important to make this non-blocking export.
- */
-public class KafkaSinkProvider implements ExternalSinkProvider {
- private static String TOPIC_NAME = "ambari-metrics-topic";
- private static final Log LOG = LogFactory.getLog(KafkaSinkProvider.class);
-
- private Producer producer;
- private int TIMEOUT_SECONDS = 10;
- private int FLUSH_SECONDS = 3;
-
- ObjectMapper objectMapper = new ObjectMapper();
-
- public KafkaSinkProvider() {
- TimelineMetricConfiguration configuration = TimelineMetricConfiguration.getInstance();
-
- Properties configProperties = new Properties();
- try {
- configProperties.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, configuration.getMetricsConf().getTrimmed(KAFKA_SERVERS));
- configProperties.put(ProducerConfig.ACKS_CONFIG, configuration.getMetricsConf().getTrimmed(KAFKA_ACKS, "all"));
- // Avoid duplicates - No transactional semantics
- configProperties.put(ProducerConfig.RETRIES_CONFIG, configuration.getMetricsConf().getInt(KAFKA_RETRIES, 0));
- configProperties.put(ProducerConfig.BATCH_SIZE_CONFIG, configuration.getMetricsConf().getInt(KAFKA_BATCH_SIZE, 128));
- configProperties.put(ProducerConfig.LINGER_MS_CONFIG, configuration.getMetricsConf().getInt(KAFKA_LINGER_MS, 1));
- configProperties.put(ProducerConfig.BUFFER_MEMORY_CONFIG, configuration.getMetricsConf().getLong(KAFKA_BUFFER_MEM, 33554432)); // 32 MB
- FLUSH_SECONDS = configuration.getMetricsConf().getInt(TIMELINE_METRICS_CACHE_COMMIT_INTERVAL, 3);
- TIMEOUT_SECONDS = configuration.getMetricsConf().getInt(KAFKA_SINK_TIMEOUT_SECONDS, 10);
- } catch (Exception e) {
- LOG.error("Configuration error!", e);
- throw new ExceptionInInitializerError(e);
- }
- configProperties.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG,"org.apache.kafka.common.serialization.ByteArraySerializer");
- configProperties.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG,"org.apache.kafka.connect.json.JsonSerializer");
-
-
-
- producer = new KafkaProducer(configProperties);
- }
-
- @Override
- public ExternalMetricsSink getExternalMetricsSink(SOURCE_NAME sourceName) {
- switch (sourceName) {
- case RAW_METRICS:
- return new KafkaRawMetricsSink();
- default:
- throw new UnsupportedOperationException("Provider does not support " +
- "the expected source " + sourceName);
- }
- }
-
- class KafkaRawMetricsSink implements ExternalMetricsSink {
-
- @Override
- public int getSinkTimeOutSeconds() {
- return TIMEOUT_SECONDS;
- }
-
- @Override
- public int getFlushSeconds() {
- return FLUSH_SECONDS;
- }
-
- @Override
- public void sinkMetricData(Collection<TimelineMetrics> metrics) {
- JsonNode jsonNode = objectMapper.valueToTree(metrics);
- ProducerRecord<String, JsonNode> rec = new ProducerRecord<String, JsonNode>(TOPIC_NAME, jsonNode);
- Future<RecordMetadata> f = producer.send(rec);
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/source/DefaultInternalMetricsSourceProvider.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/source/DefaultInternalMetricsSourceProvider.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/source/DefaultInternalMetricsSourceProvider.java
deleted file mode 100644
index c6b071f..0000000
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/source/DefaultInternalMetricsSourceProvider.java
+++ /dev/null
@@ -1,42 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.source;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.sink.ExternalMetricsSink;
-
-public class DefaultInternalMetricsSourceProvider implements InternalSourceProvider {
- private static final Log LOG = LogFactory.getLog(DefaultInternalMetricsSourceProvider.class);
-
- // TODO: Implement read based sources for higher order data
- @Override
- public InternalMetricsSource getInternalMetricsSource(SOURCE_NAME sourceName, int sinkIntervalSeconds, ExternalMetricsSink sink) {
- if (sink == null) {
- LOG.warn("No external sink configured for source " + sourceName);
- return null;
- }
-
- switch (sourceName) {
- case RAW_METRICS:
- return new RawMetricsSource(sinkIntervalSeconds, sink);
- default:
- throw new UnsupportedOperationException("Unimplemented source type " + sourceName);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/source/InternalMetricsSource.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/source/InternalMetricsSource.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/source/InternalMetricsSource.java
deleted file mode 100644
index a6e1092..0000000
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/source/InternalMetricsSource.java
+++ /dev/null
@@ -1,30 +0,0 @@
-package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.source;
-
-import java.util.Collection;
-
-import org.apache.hadoop.metrics2.sink.timeline.TimelineMetrics;
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-public interface InternalMetricsSource {
- /**
- * Write metrics to external sink.
- * Allows pre-processing and caching capabilities to the consumer.
- */
- void publishTimelineMetrics(Collection<TimelineMetrics> metrics);
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/source/InternalSourceProvider.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/source/InternalSourceProvider.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/source/InternalSourceProvider.java
deleted file mode 100644
index 9d8ca36..0000000
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/source/InternalSourceProvider.java
+++ /dev/null
@@ -1,39 +0,0 @@
-package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.source;
-
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.sink.ExternalMetricsSink;
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-public interface InternalSourceProvider {
-
- enum SOURCE_NAME {
- RAW_METRICS,
- MINUTE_HOST_AGGREAGATE_METRICS,
- HOURLY_HOST_AGGREAGATE_METRICS,
- DAILY_HOST_AGGREAGATE_METRICS,
- MINUTE_CLUSTER_AGGREAGATE_METRICS,
- HOURLY_CLUSTER_AGGREAGATE_METRICS,
- DAILY_CLUSTER_AGGREAGATE_METRICS,
- }
-
- /**
- * Provide Source for metrics data.
- * @return {@link InternalMetricsSource}
- */
- InternalMetricsSource getInternalMetricsSource(SOURCE_NAME sourceName, int sinkIntervalSeconds, ExternalMetricsSink sink);
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/source/RawMetricsSource.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/source/RawMetricsSource.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/source/RawMetricsSource.java
deleted file mode 100644
index 6475536..0000000
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/source/RawMetricsSource.java
+++ /dev/null
@@ -1,85 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.source;
-
-import java.util.Collection;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.Executors;
-import java.util.concurrent.Future;
-import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.TimeoutException;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.metrics2.sink.timeline.TimelineMetrics;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.sink.ExternalMetricsSink;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.source.cache.InternalMetricsCache;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.source.cache.InternalMetricsCacheProvider;
-
-public class RawMetricsSource implements InternalMetricsSource {
- private static final Log LOG = LogFactory.getLog(RawMetricsSource.class);
- private final int internalCacheInterval;
- private final ExternalMetricsSink rawMetricsSink;
- private final ScheduledExecutorService executorService = Executors.newSingleThreadScheduledExecutor();
- private final InternalMetricsCache cache;
- static final String RAW_METRICS_CACHE = "RAW_METRICS_CACHE_INSTANCE";
-
- public RawMetricsSource(int internalCacheInterval, ExternalMetricsSink rawMetricsSink) {
- this.internalCacheInterval = internalCacheInterval;
- this.rawMetricsSink = rawMetricsSink;
- this.cache = InternalMetricsCacheProvider.getInstance().getCacheInstance(RAW_METRICS_CACHE);
- if (rawMetricsSink.getFlushSeconds() > internalCacheInterval) {
- initializeFixedRateScheduler();
- }
- }
-
- @Override
- public void publishTimelineMetrics(Collection<TimelineMetrics> metrics) {
- // TODO: Adjust default flush to reasonable defaults > 3 seconds
- if (rawMetricsSink.getFlushSeconds() > internalCacheInterval) {
- // Need to cache only if external sink cannot keep up and thereby has
- // different flush interval as compared to HBase flush
- cache.putAll(metrics); // Scheduler initialized already for flush
- } else {
- submitDataWithTimeout(metrics);
- }
- }
-
- private void initializeFixedRateScheduler() {
- executorService.scheduleAtFixedRate(() -> rawMetricsSink.sinkMetricData(cache.evictAll()),
- rawMetricsSink.getFlushSeconds(), rawMetricsSink.getFlushSeconds(), TimeUnit.SECONDS);
- }
-
- private void submitDataWithTimeout(final Collection<TimelineMetrics> metrics) {
- Future f = executorService.submit(() -> {
- rawMetricsSink.sinkMetricData(metrics);
- return null;
- });
- try {
- f.get(rawMetricsSink.getSinkTimeOutSeconds(), TimeUnit.SECONDS);
- } catch (InterruptedException e) {
- LOG.warn("Raw metrics sink interrupted.");
- } catch (ExecutionException e) {
- LOG.warn("Exception on sinking metrics", e);
- } catch (TimeoutException e) {
- LOG.warn("Timeout exception on sinking metrics", e);
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/source/cache/InternalMetricCacheKey.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/source/cache/InternalMetricCacheKey.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/source/cache/InternalMetricCacheKey.java
deleted file mode 100644
index 28d457d..0000000
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/source/cache/InternalMetricCacheKey.java
+++ /dev/null
@@ -1,109 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.source.cache;
-
-public class InternalMetricCacheKey {
- private String metricName;
- private String appId;
- private String instanceId;
- private String hostname;
- private long startTime; // Useful for debugging
-
- public InternalMetricCacheKey(String metricName, String appId, String instanceId, String hostname, long startTime) {
- this.metricName = metricName;
- this.appId = appId;
- this.instanceId = instanceId;
- this.hostname = hostname;
- this.startTime = startTime;
- }
-
- public String getMetricName() {
- return metricName;
- }
-
- public void setMetricName(String metricName) {
- this.metricName = metricName;
- }
-
- public String getAppId() {
- return appId;
- }
-
- public void setAppId(String appId) {
- this.appId = appId;
- }
-
- public String getInstanceId() {
- return instanceId;
- }
-
- public void setInstanceId(String instanceId) {
- this.instanceId = instanceId;
- }
-
- public String getHostname() {
- return hostname;
- }
-
- public void setHostname(String hostname) {
- this.hostname = hostname;
- }
-
- public long getStartTime() {
- return startTime;
- }
-
- public void setStartTime(long startTime) {
- this.startTime = startTime;
- }
-
- @Override
- public boolean equals(Object o) {
- if (this == o) return true;
- if (o == null || getClass() != o.getClass()) return false;
-
- InternalMetricCacheKey that = (InternalMetricCacheKey) o;
-
- if (!getMetricName().equals(that.getMetricName())) return false;
- if (!getAppId().equals(that.getAppId())) return false;
- if (getInstanceId() != null ? !getInstanceId().equals(that.getInstanceId()) : that.getInstanceId() != null)
- return false;
- return getHostname() != null ? getHostname().equals(that.getHostname()) : that.getHostname() == null;
-
- }
-
- @Override
- public int hashCode() {
- int result = getMetricName().hashCode();
- result = 31 * result + getAppId().hashCode();
- result = 31 * result + (getInstanceId() != null ? getInstanceId().hashCode() : 0);
- result = 31 * result + (getHostname() != null ? getHostname().hashCode() : 0);
- return result;
- }
-
- @Override
- public String toString() {
- return "InternalMetricCacheKey{" +
- "metricName='" + metricName + '\'' +
- ", appId='" + appId + '\'' +
- ", instanceId='" + instanceId + '\'' +
- ", hostname='" + hostname + '\'' +
- ", startTime=" + startTime +
- '}';
- }
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/source/cache/InternalMetricCacheValue.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/source/cache/InternalMetricCacheValue.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/source/cache/InternalMetricCacheValue.java
deleted file mode 100644
index a4dabe7..0000000
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/source/cache/InternalMetricCacheValue.java
+++ /dev/null
@@ -1,37 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.source.cache;
-
-import java.util.Map;
-import java.util.TreeMap;
-
-public class InternalMetricCacheValue {
- private TreeMap<Long, Double> metricValues = new TreeMap<Long, Double>();
-
- public TreeMap<Long, Double> getMetricValues() {
- return metricValues;
- }
-
- public void setMetricValues(TreeMap<Long, Double> metricValues) {
- this.metricValues = metricValues;
- }
-
- public void addMetricValues(Map<Long, Double> metricValues) {
- this.metricValues.putAll(metricValues);
- }
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/source/cache/InternalMetricsCache.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/source/cache/InternalMetricsCache.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/source/cache/InternalMetricsCache.java
deleted file mode 100644
index e5522c7..0000000
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/source/cache/InternalMetricsCache.java
+++ /dev/null
@@ -1,229 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.source.cache;
-
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Date;
-import java.util.List;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.locks.Lock;
-import java.util.concurrent.locks.ReentrantLock;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.metrics2.sink.timeline.TimelineMetric;
-import org.apache.hadoop.metrics2.sink.timeline.TimelineMetrics;
-
-import net.sf.ehcache.Cache;
-import net.sf.ehcache.CacheException;
-import net.sf.ehcache.CacheManager;
-import net.sf.ehcache.Ehcache;
-import net.sf.ehcache.Element;
-import net.sf.ehcache.config.CacheConfiguration;
-import net.sf.ehcache.config.PersistenceConfiguration;
-import net.sf.ehcache.config.SizeOfPolicyConfiguration;
-import net.sf.ehcache.event.CacheEventListener;
-import net.sf.ehcache.store.MemoryStoreEvictionPolicy;
-
-public class InternalMetricsCache {
- private static final Log LOG = LogFactory.getLog(InternalMetricsCache.class);
- private final String instanceName;
- private final String maxHeapPercent;
- private volatile boolean isCacheInitialized = false;
- private Cache cache;
- static final String TIMELINE_METRIC_CACHE_MANAGER_NAME = "internalMetricsCacheManager";
- private final Lock lock = new ReentrantLock();
- private static final int LOCK_TIMEOUT_SECONDS = 2;
-
- public InternalMetricsCache(String instanceName, String maxHeapPercent) {
- this.instanceName = instanceName;
- this.maxHeapPercent = maxHeapPercent;
- initialize();
- }
-
- private void initialize() {
- // Check in case of contention to avoid ObjectExistsException
- if (isCacheInitialized) {
- throw new RuntimeException("Cannot initialize internal cache twice");
- }
-
- System.setProperty("net.sf.ehcache.skipUpdateCheck", "true");
- System.setProperty("net.sf.ehcache.sizeofengine." + TIMELINE_METRIC_CACHE_MANAGER_NAME,
- "org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.source.cache.InternalMetricsCacheSizeOfEngine");
-
- net.sf.ehcache.config.Configuration managerConfig =
- new net.sf.ehcache.config.Configuration();
- managerConfig.setName(TIMELINE_METRIC_CACHE_MANAGER_NAME);
-
- // Set max heap available to the cache manager
- managerConfig.setMaxBytesLocalHeap(maxHeapPercent);
-
- //Create a singleton CacheManager using defaults
- CacheManager manager = CacheManager.create(managerConfig);
-
- LOG.info("Creating Metrics Cache with maxHeapPercent => " + maxHeapPercent);
-
- // Create a Cache specifying its configuration.
- CacheConfiguration cacheConfiguration = new CacheConfiguration()
- .name(instanceName)
- .memoryStoreEvictionPolicy(MemoryStoreEvictionPolicy.LRU)
- .sizeOfPolicy(new SizeOfPolicyConfiguration() // Set sizeOf policy to continue on max depth reached - avoid OOM
- .maxDepth(10000)
- .maxDepthExceededBehavior(SizeOfPolicyConfiguration.MaxDepthExceededBehavior.CONTINUE))
- .eternal(true) // infinite time until eviction
- .persistence(new PersistenceConfiguration()
- .strategy(PersistenceConfiguration.Strategy.NONE.name()));
-
- cache = new Cache(cacheConfiguration);
- cache.getCacheEventNotificationService().registerListener(new InternalCacheEvictionListener());
-
- LOG.info("Registering internal metrics cache with provider: name = " +
- cache.getName() + ", guid: " + cache.getGuid());
-
- manager.addCache(cache);
-
- isCacheInitialized = true;
- }
-
- public InternalMetricCacheValue getInternalMetricCacheValue(InternalMetricCacheKey key) {
- Element ele = cache.get(key);
- if (ele != null) {
- return (InternalMetricCacheValue) ele.getObjectValue();
- }
- return null;
- }
-
- public Collection<TimelineMetrics> evictAll() {
- TimelineMetrics metrics = new TimelineMetrics();
- try {
- if (lock.tryLock(LOCK_TIMEOUT_SECONDS, TimeUnit.SECONDS)) {
- try{
- List keys = cache.getKeys();
- for (Object obj : keys) {
- TimelineMetric metric = new TimelineMetric();
- InternalMetricCacheKey key = (InternalMetricCacheKey) obj;
- metric.setMetricName(key.getMetricName());
- metric.setAppId(key.getAppId());
- metric.setInstanceId(key.getInstanceId());
- metric.setHostName(key.getHostname());
- metric.setStartTime(key.getStartTime());
- Element ele = cache.get(key);
- metric.setMetricValues(((InternalMetricCacheValue) ele.getObjectValue()).getMetricValues());
- metrics.getMetrics().add(metric);
- }
- cache.removeAll();
- } finally {
- lock.unlock();
- }
- } else {
- LOG.warn("evictAll: Unable to acquire lock on the cache instance. " +
- "Giving up after " + LOCK_TIMEOUT_SECONDS + " seconds.");
- }
- } catch (InterruptedException e) {
- LOG.warn("Interrupted while waiting to acquire lock");
- }
-
- return Collections.singletonList(metrics);
- }
-
- public void putAll(Collection<TimelineMetrics> metrics) {
- try {
- if (lock.tryLock(LOCK_TIMEOUT_SECONDS, TimeUnit.SECONDS)) {
- try {
- if (metrics != null) {
- for (TimelineMetrics timelineMetrics : metrics) {
- for (TimelineMetric timelineMetric : timelineMetrics.getMetrics()) {
- InternalMetricCacheKey key = new InternalMetricCacheKey(
- timelineMetric.getMetricName(),
- timelineMetric.getAppId(),
- timelineMetric.getInstanceId(),
- timelineMetric.getHostName(),
- timelineMetric.getStartTime()
- );
-
- Element ele = cache.get(key);
- if (ele != null) {
- InternalMetricCacheValue value = (InternalMetricCacheValue) ele.getObjectValue();
- value.addMetricValues(timelineMetric.getMetricValues());
- } else {
- InternalMetricCacheValue value = new InternalMetricCacheValue();
- value.setMetricValues(timelineMetric.getMetricValues());
- cache.put(new Element(key, value));
- }
- }
- }
- }
- } finally {
- lock.unlock();
- }
- } else {
- LOG.warn("putAll: Unable to acquire lock on the cache instance. " +
- "Giving up after " + LOCK_TIMEOUT_SECONDS + " seconds.");
- }
- } catch (InterruptedException e) {
- LOG.warn("Interrupted while waiting to acquire lock");
- }
- }
-
- class InternalCacheEvictionListener implements CacheEventListener {
-
- @Override
- public void notifyElementRemoved(Ehcache cache, Element element) throws CacheException {
- // expected
- }
-
- @Override
- public void notifyElementPut(Ehcache cache, Element element) throws CacheException {
- // do nothing
- }
-
- @Override
- public void notifyElementUpdated(Ehcache cache, Element element) throws CacheException {
- // do nothing
- }
-
- @Override
- public void notifyElementExpired(Ehcache cache, Element element) {
- // do nothing
- }
-
- @Override
- public void notifyElementEvicted(Ehcache cache, Element element) {
- // Bad - Remote endpoint cannot keep up resulting in flooding
- InternalMetricCacheKey key = (InternalMetricCacheKey) element.getObjectKey();
- LOG.warn("Evicting element from internal metrics cache, metric => " + key
- .getMetricName() + ", startTime = " + new Date(key.getStartTime()));
- }
-
- @Override
- public void notifyRemoveAll(Ehcache cache) {
- // expected
- }
-
- @Override
- public Object clone() throws CloneNotSupportedException {
- return null;
- }
-
- @Override
- public void dispose() {
- // do nothing
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/source/cache/InternalMetricsCacheProvider.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/source/cache/InternalMetricsCacheProvider.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/source/cache/InternalMetricsCacheProvider.java
deleted file mode 100644
index 3e0dc1b..0000000
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/source/cache/InternalMetricsCacheProvider.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.source.cache;
-
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
-
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration;
-
-public class InternalMetricsCacheProvider {
- private Map<String, InternalMetricsCache> metricsCacheMap = new ConcurrentHashMap<>();
- private static final InternalMetricsCacheProvider instance = new InternalMetricsCacheProvider();
-
- private InternalMetricsCacheProvider() {
- }
-
- public static InternalMetricsCacheProvider getInstance() {
- return instance;
- }
-
- public InternalMetricsCache getCacheInstance(String instanceName) {
- if (metricsCacheMap.containsKey(instanceName)) {
- return metricsCacheMap.get(instanceName);
- } else {
- TimelineMetricConfiguration conf = TimelineMetricConfiguration.getInstance();
- InternalMetricsCache cache = new InternalMetricsCache(instanceName,
- conf.getInternalCacheHeapPercent(instanceName));
-
- metricsCacheMap.put(instanceName, cache);
- return cache;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/source/cache/InternalMetricsCacheSizeOfEngine.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/source/cache/InternalMetricsCacheSizeOfEngine.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/source/cache/InternalMetricsCacheSizeOfEngine.java
deleted file mode 100644
index e36c981..0000000
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/source/cache/InternalMetricsCacheSizeOfEngine.java
+++ /dev/null
@@ -1,52 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.source.cache;
-
-import org.apache.hadoop.metrics2.sink.timeline.cache.TimelineMetricsEhCacheSizeOfEngine;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class InternalMetricsCacheSizeOfEngine extends TimelineMetricsEhCacheSizeOfEngine {
- private final static Logger LOG = LoggerFactory.getLogger(InternalMetricsCacheSizeOfEngine.class);
-
- public InternalMetricsCacheSizeOfEngine() {
- // Invoke default constructor in base class
- }
-
- @Override
- protected long getSizeOfEntry(Object key, Object value) {
- try {
- LOG.debug("BEGIN - Sizeof, key: {}, value: {}", key, value);
- long size = 0;
- if (key instanceof InternalMetricCacheKey) {
- InternalMetricCacheKey metricCacheKey = (InternalMetricCacheKey) key;
- size += reflectionSizeOf.sizeOf(metricCacheKey.getMetricName());
- size += reflectionSizeOf.sizeOf(metricCacheKey.getAppId());
- size += reflectionSizeOf.sizeOf(metricCacheKey.getInstanceId()); // null safe
- size += reflectionSizeOf.sizeOf(metricCacheKey.getHostname());
- }
- if (value instanceof InternalMetricCacheValue) {
- size += getValueMapSize(((InternalMetricCacheValue) value).getMetricValues());
- }
- // Mark size as not being exact
- return size;
- } finally {
- LOG.debug("END - Sizeof, key: {}", key);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/uuid/HashBasedUuidGenStrategy.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/uuid/HashBasedUuidGenStrategy.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/uuid/HashBasedUuidGenStrategy.java
deleted file mode 100644
index 3acf656..0000000
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/uuid/HashBasedUuidGenStrategy.java
+++ /dev/null
@@ -1,206 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.uuid;
-
-import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-
-import org.apache.commons.lang.ArrayUtils;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.aggregators.TimelineClusterMetric;
-
-public class HashBasedUuidGenStrategy implements MetricUuidGenStrategy {
-
- /**
- * Computes the UUID for a timelineClusterMetric.
- * @param timelineClusterMetric
- * @param maxLength
- * @return byte array of length 'maxlength'
- */
- @Override
- public byte[] computeUuid(TimelineClusterMetric timelineClusterMetric, int maxLength) {
-
- int metricNameUuidLength = 12;
- String metricName = timelineClusterMetric.getMetricName();
-
- //Compute the individual splits.
- String[] splits = getIndidivualSplits(metricName);
-
- /*
- Compute the ascii sum of every split in the metric name. (asciiSum += (int) splits[s].charAt(i))
- For the last split, use weighted sum instead of ascii sum. (asciiSum += ((i+1) * (int) splits[s].charAt(i)))
- These weighted sums are 'appended' to get the unique ID for metric name.
- */
- StringBuilder splitSums = new StringBuilder();
- if (splits.length > 0) {
- for (int s = 0; s < splits.length; s++) {
- int asciiSum = 0;
- if ( s < splits.length -1) {
- for (int i = 0; i < splits[s].length(); i++) {
- asciiSum += (int) splits[s].charAt(i); // Get Ascii Sum.
- }
- } else {
- for (int i = 0; i < splits[s].length(); i++) {
- asciiSum += ((i+1) * (int) splits[s].charAt(i)); //weighted sum for last split.
- }
- }
- splitSums.append(asciiSum); //Append the sum to the array of sums.
- }
- }
-
- //Compute a unique metric seed for the stemmed metric name
- String stemmedMetric = stem(metricName);
- long metricSeed = 100123456789L;
- for (int i = 0; i < stemmedMetric.length(); i++) {
- metricSeed += stemmedMetric.charAt(i);
- }
-
- //Reverse the computed seed to get a metric UUID portion which is used optionally.
- byte[] metricUuidPortion = StringUtils.reverse(String.valueOf(metricSeed)).getBytes();
- String splitSumString = splitSums.toString();
- int splitLength = splitSumString.length();
-
- //If splitSums length > required metric UUID length, use only the required length suffix substring of the splitSums as metric UUID.
- if (splitLength > metricNameUuidLength) {
- metricUuidPortion = ArrayUtils.subarray(splitSumString.getBytes(), splitLength - metricNameUuidLength, splitLength);
- } else {
- //If splitSums is not enough for required metric UUID length, pad with the metric uuid portion.
- int pad = metricNameUuidLength - splitLength;
- metricUuidPortion = ArrayUtils.addAll(splitSumString.getBytes(), ArrayUtils.subarray(metricUuidPortion, 0, pad));
- }
-
- /*
- For appId and instanceId the logic is similar. Use a seed integer to start with and compute ascii sum.
- Based on required length, use a suffix of the computed uuid.
- */
- String appId = timelineClusterMetric.getAppId();
- int appidSeed = 11;
- for (int i = 0; i < appId.length(); i++) {
- appidSeed += appId.charAt(i);
- }
- String appIdSeedStr = String.valueOf(appidSeed);
- byte[] appUuidPortion = ArrayUtils.subarray(appIdSeedStr.getBytes(), appIdSeedStr.length() - 2, appIdSeedStr.length());
-
- String instanceId = timelineClusterMetric.getInstanceId();
- ByteBuffer buffer = ByteBuffer.allocate(4);
- byte[] instanceUuidPortion = new byte[2];
- if (StringUtils.isNotEmpty(instanceId)) {
- int instanceIdSeed = 1489;
- for (int i = 0; i < appId.length(); i++) {
- instanceIdSeed += appId.charAt(i);
- }
- buffer.putInt(instanceIdSeed);
- ArrayUtils.subarray(buffer.array(), 2, 4);
- }
-
- // Concatenate all UUIDs together (metric uuid + appId uuid + instanceId uuid)
- return ArrayUtils.addAll(ArrayUtils.addAll(metricUuidPortion, appUuidPortion), instanceUuidPortion);
- }
-
- /**
- * Splits the metric name into individual tokens.
- * For example,
- * kafka.server.ReplicaManager.LeaderCount -> [kafka, server, ReplicaManager, LeaderCount]
- * default.General.api_drop_table_15min_rate -> [default, General, api, drop, table, 15min, rate]
- * @param metricName
- * @return
- */
- private String[] getIndidivualSplits(String metricName) {
- List<String> tokens = new ArrayList<>();
- String[] splits = new String[0];
- if (metricName.contains("\\.")) {
- splits = metricName.split("\\.");
- for (String split : splits) {
- if (split.contains("_")) {
- tokens.addAll(Arrays.asList(split.split("_")));
- } else {
- tokens.add(split);
- }
- }
- }
-
- if (splits.length <= 1) {
- splits = metricName.split("\\_");
- return splits;
- }
-
- if (splits.length <= 1) {
- splits = metricName.split("\\=");
- return splits;
- }
-
- return tokens.toArray(new String[tokens.size()]);
- }
-
- /**
- * Stem the metric name. Remove a set of usual suspects characters.
- * @param metricName
- * @return
- */
- private String stem(String metricName) {
- String metric = metricName.toLowerCase();
- String regex = "[\\.\\_\\%\\-\\=]";
- String trimmedMetric = StringUtils.removePattern(metric, regex);
- return trimmedMetric;
- }
-
-
- /**
- * Computes the UUID of a string. (hostname)
- * Uses the ascii sum of the String. Numbers in the String are treated as actual numerical values rather than ascii values.
- * @param value
- * @param maxLength
- * @return byte array of length 'maxlength'
- */
- @Override
- public byte[] computeUuid(String value, int maxLength) {
-
- if (StringUtils.isEmpty(value)) {
- return null;
- }
- int len = value.length();
- int numericValue = 0;
- int seed = 1489;
- for (int i = 0; i < len; i++) {
- int ascii = value.charAt(i);
- if (48 <= ascii && ascii <= 57) {
- numericValue += numericValue * 10 + (ascii - 48);
- } else {
- if (numericValue > 0) {
- seed += numericValue;
- numericValue = 0;
- }
- seed+= value.charAt(i);
- }
- }
-
- if (numericValue != 0) {
- seed+=numericValue;
- }
-
- String seedStr = String.valueOf(seed);
- if (seedStr.length() < maxLength) {
- return null;
- } else {
- return seedStr.substring(seedStr.length() - maxLength, seedStr.length()).getBytes();
- }
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/uuid/MetricUuidGenStrategy.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/uuid/MetricUuidGenStrategy.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/uuid/MetricUuidGenStrategy.java
deleted file mode 100644
index b6a1890..0000000
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/uuid/MetricUuidGenStrategy.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.uuid;
-
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.aggregators.TimelineClusterMetric;
-
-public interface MetricUuidGenStrategy {
-
- /**
- * Compute UUID for a given value
- * @param timelineMetric instance
- * @param maxLength
- * @return
- */
-// byte[] computeUuid(TimelineMetric timelineMetric, int maxLength);
-
- /**
- * Compute UUID for a given value
- * @param value
- * @param maxLength
- * @return
- */
- byte[] computeUuid(TimelineClusterMetric timelineClusterMetric, int maxLength);
-
- /**
- * Compute UUID for a given value
- * @param value
- * @param maxLength
- * @return
- */
- byte[] computeUuid(String value, int maxLength);
-
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/uuid/RandomUuidGenStrategy.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/uuid/RandomUuidGenStrategy.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/uuid/RandomUuidGenStrategy.java
deleted file mode 100644
index 1443067..0000000
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/uuid/RandomUuidGenStrategy.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.uuid;
-
-import java.security.SecureRandom;
-
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.aggregators.TimelineClusterMetric;
-
-import com.google.common.primitives.Longs;
-
-public class RandomUuidGenStrategy implements MetricUuidGenStrategy {
- private static SecureRandom randomGenerator;
-
- public RandomUuidGenStrategy() {
- randomGenerator = new SecureRandom(
- Longs.toByteArray(System.currentTimeMillis()));
- }
-
- @Override
- public byte[] computeUuid(TimelineClusterMetric timelineClusterMetric, int maxLength) {
- final byte[] bytes = new byte[maxLength];
- randomGenerator.nextBytes(bytes);
- return bytes;
- }
-
-// @Override
-// public byte[] computeUuid(TimelineMetric timelineMetric, int maxLength) {
-// return new byte[10];
-// }
-
- @Override
- public byte[] computeUuid(String value, int maxLength) {
- final byte[] bytes = new byte[maxLength];
- randomGenerator.nextBytes(bytes);
- return bytes;
- }
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/ApplicationAttemptFinishData.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/ApplicationAttemptFinishData.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/ApplicationAttemptFinishData.java
deleted file mode 100644
index 7ba51af..0000000
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/ApplicationAttemptFinishData.java
+++ /dev/null
@@ -1,95 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.server.applicationhistoryservice.records;
-
-import org.apache.hadoop.classification.InterfaceAudience.Public;
-import org.apache.hadoop.classification.InterfaceStability.Unstable;
-import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
-import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
-import org.apache.hadoop.yarn.api.records.YarnApplicationAttemptState;
-import org.apache.hadoop.yarn.util.Records;
-
-/**
- * The class contains the fields that can be determined when
- * <code>RMAppAttempt</code> finishes, and that need to be stored persistently.
- */
-@Public
-@Unstable
-public abstract class ApplicationAttemptFinishData {
-
- @Public
- @Unstable
- public static ApplicationAttemptFinishData newInstance(
- ApplicationAttemptId appAttemptId, String diagnosticsInfo,
- String trackingURL, FinalApplicationStatus finalApplicationStatus,
- YarnApplicationAttemptState yarnApplicationAttemptState) {
- ApplicationAttemptFinishData appAttemptFD =
- Records.newRecord(ApplicationAttemptFinishData.class);
- appAttemptFD.setApplicationAttemptId(appAttemptId);
- appAttemptFD.setDiagnosticsInfo(diagnosticsInfo);
- appAttemptFD.setTrackingURL(trackingURL);
- appAttemptFD.setFinalApplicationStatus(finalApplicationStatus);
- appAttemptFD.setYarnApplicationAttemptState(yarnApplicationAttemptState);
- return appAttemptFD;
- }
-
- @Public
- @Unstable
- public abstract ApplicationAttemptId getApplicationAttemptId();
-
- @Public
- @Unstable
- public abstract void setApplicationAttemptId(
- ApplicationAttemptId applicationAttemptId);
-
- @Public
- @Unstable
- public abstract String getTrackingURL();
-
- @Public
- @Unstable
- public abstract void setTrackingURL(String trackingURL);
-
- @Public
- @Unstable
- public abstract String getDiagnosticsInfo();
-
- @Public
- @Unstable
- public abstract void setDiagnosticsInfo(String diagnosticsInfo);
-
- @Public
- @Unstable
- public abstract FinalApplicationStatus getFinalApplicationStatus();
-
- @Public
- @Unstable
- public abstract void setFinalApplicationStatus(
- FinalApplicationStatus finalApplicationStatus);
-
- @Public
- @Unstable
- public abstract YarnApplicationAttemptState getYarnApplicationAttemptState();
-
- @Public
- @Unstable
- public abstract void setYarnApplicationAttemptState(
- YarnApplicationAttemptState yarnApplicationAttemptState);
-
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/ApplicationAttemptHistoryData.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/ApplicationAttemptHistoryData.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/ApplicationAttemptHistoryData.java
deleted file mode 100644
index b759ab1..0000000
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/ApplicationAttemptHistoryData.java
+++ /dev/null
@@ -1,171 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.server.applicationhistoryservice.records;
-
-import org.apache.hadoop.classification.InterfaceAudience.Public;
-import org.apache.hadoop.classification.InterfaceStability.Unstable;
-import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
-import org.apache.hadoop.yarn.api.records.YarnApplicationAttemptState;
-
-/**
- * The class contains all the fields that are stored persistently for
- * <code>RMAppAttempt</code>.
- */
-@Public
-@Unstable
-public class ApplicationAttemptHistoryData {
-
- private ApplicationAttemptId applicationAttemptId;
-
- private String host;
-
- private int rpcPort;
-
- private String trackingURL;
-
- private String diagnosticsInfo;
-
- private FinalApplicationStatus finalApplicationStatus;
-
- private ContainerId masterContainerId;
-
- private YarnApplicationAttemptState yarnApplicationAttemptState;
-
- @Public
- @Unstable
- public static ApplicationAttemptHistoryData newInstance(
- ApplicationAttemptId appAttemptId, String host, int rpcPort,
- ContainerId masterContainerId, String diagnosticsInfo,
- String trackingURL, FinalApplicationStatus finalApplicationStatus,
- YarnApplicationAttemptState yarnApplicationAttemptState) {
- ApplicationAttemptHistoryData appAttemptHD =
- new ApplicationAttemptHistoryData();
- appAttemptHD.setApplicationAttemptId(appAttemptId);
- appAttemptHD.setHost(host);
- appAttemptHD.setRPCPort(rpcPort);
- appAttemptHD.setMasterContainerId(masterContainerId);
- appAttemptHD.setDiagnosticsInfo(diagnosticsInfo);
- appAttemptHD.setTrackingURL(trackingURL);
- appAttemptHD.setFinalApplicationStatus(finalApplicationStatus);
- appAttemptHD.setYarnApplicationAttemptState(yarnApplicationAttemptState);
- return appAttemptHD;
- }
-
- @Public
- @Unstable
- public ApplicationAttemptId getApplicationAttemptId() {
- return applicationAttemptId;
- }
-
- @Public
- @Unstable
- public void
- setApplicationAttemptId(ApplicationAttemptId applicationAttemptId) {
- this.applicationAttemptId = applicationAttemptId;
- }
-
- @Public
- @Unstable
- public String getHost() {
- return host;
- }
-
- @Public
- @Unstable
- public void setHost(String host) {
- this.host = host;
- }
-
- @Public
- @Unstable
- public int getRPCPort() {
- return rpcPort;
- }
-
- @Public
- @Unstable
- public void setRPCPort(int rpcPort) {
- this.rpcPort = rpcPort;
- }
-
- @Public
- @Unstable
- public String getTrackingURL() {
- return trackingURL;
- }
-
- @Public
- @Unstable
- public void setTrackingURL(String trackingURL) {
- this.trackingURL = trackingURL;
- }
-
- @Public
- @Unstable
- public String getDiagnosticsInfo() {
- return diagnosticsInfo;
- }
-
- @Public
- @Unstable
- public void setDiagnosticsInfo(String diagnosticsInfo) {
- this.diagnosticsInfo = diagnosticsInfo;
- }
-
- @Public
- @Unstable
- public FinalApplicationStatus getFinalApplicationStatus() {
- return finalApplicationStatus;
- }
-
- @Public
- @Unstable
- public void setFinalApplicationStatus(
- FinalApplicationStatus finalApplicationStatus) {
- this.finalApplicationStatus = finalApplicationStatus;
- }
-
- @Public
- @Unstable
- public ContainerId getMasterContainerId() {
- return masterContainerId;
- }
-
- @Public
- @Unstable
- public void setMasterContainerId(ContainerId masterContainerId) {
- this.masterContainerId = masterContainerId;
- }
-
- @Public
- @Unstable
- public YarnApplicationAttemptState getYarnApplicationAttemptState() {
- return yarnApplicationAttemptState;
- }
-
- @Public
- @Unstable
- public void setYarnApplicationAttemptState(
- YarnApplicationAttemptState yarnApplicationAttemptState) {
- this.yarnApplicationAttemptState = yarnApplicationAttemptState;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/ApplicationAttemptStartData.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/ApplicationAttemptStartData.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/ApplicationAttemptStartData.java
deleted file mode 100644
index 7ca43fa..0000000
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/ApplicationAttemptStartData.java
+++ /dev/null
@@ -1,82 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.server.applicationhistoryservice.records;
-
-import org.apache.hadoop.classification.InterfaceAudience.Public;
-import org.apache.hadoop.classification.InterfaceStability.Unstable;
-import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.util.Records;
-
-/**
- * The class contains the fields that can be determined when
- * <code>RMAppAttempt</code> starts, and that need to be stored persistently.
- */
-@Public
-@Unstable
-public abstract class ApplicationAttemptStartData {
-
- @Public
- @Unstable
- public static ApplicationAttemptStartData newInstance(
- ApplicationAttemptId appAttemptId, String host, int rpcPort,
- ContainerId masterContainerId) {
- ApplicationAttemptStartData appAttemptSD =
- Records.newRecord(ApplicationAttemptStartData.class);
- appAttemptSD.setApplicationAttemptId(appAttemptId);
- appAttemptSD.setHost(host);
- appAttemptSD.setRPCPort(rpcPort);
- appAttemptSD.setMasterContainerId(masterContainerId);
- return appAttemptSD;
- }
-
- @Public
- @Unstable
- public abstract ApplicationAttemptId getApplicationAttemptId();
-
- @Public
- @Unstable
- public abstract void setApplicationAttemptId(
- ApplicationAttemptId applicationAttemptId);
-
- @Public
- @Unstable
- public abstract String getHost();
-
- @Public
- @Unstable
- public abstract void setHost(String host);
-
- @Public
- @Unstable
- public abstract int getRPCPort();
-
- @Public
- @Unstable
- public abstract void setRPCPort(int rpcPort);
-
- @Public
- @Unstable
- public abstract ContainerId getMasterContainerId();
-
- @Public
- @Unstable
- public abstract void setMasterContainerId(ContainerId masterContainerId);
-
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/ApplicationFinishData.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/ApplicationFinishData.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/ApplicationFinishData.java
deleted file mode 100644
index 997fa6c..0000000
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/ApplicationFinishData.java
+++ /dev/null
@@ -1,94 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.server.applicationhistoryservice.records;
-
-import org.apache.hadoop.classification.InterfaceAudience.Public;
-import org.apache.hadoop.classification.InterfaceStability.Unstable;
-import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
-import org.apache.hadoop.yarn.api.records.YarnApplicationState;
-import org.apache.hadoop.yarn.util.Records;
-
-/**
- * The class contains the fields that can be determined when <code>RMApp</code>
- * finishes, and that need to be stored persistently.
- */
-@Public
-@Unstable
-public abstract class ApplicationFinishData {
-
- @Public
- @Unstable
- public static ApplicationFinishData newInstance(ApplicationId applicationId,
- long finishTime, String diagnosticsInfo,
- FinalApplicationStatus finalApplicationStatus,
- YarnApplicationState yarnApplicationState) {
- ApplicationFinishData appFD =
- Records.newRecord(ApplicationFinishData.class);
- appFD.setApplicationId(applicationId);
- appFD.setFinishTime(finishTime);
- appFD.setDiagnosticsInfo(diagnosticsInfo);
- appFD.setFinalApplicationStatus(finalApplicationStatus);
- appFD.setYarnApplicationState(yarnApplicationState);
- return appFD;
- }
-
- @Public
- @Unstable
- public abstract ApplicationId getApplicationId();
-
- @Public
- @Unstable
- public abstract void setApplicationId(ApplicationId applicationId);
-
- @Public
- @Unstable
- public abstract long getFinishTime();
-
- @Public
- @Unstable
- public abstract void setFinishTime(long finishTime);
-
- @Public
- @Unstable
- public abstract String getDiagnosticsInfo();
-
- @Public
- @Unstable
- public abstract void setDiagnosticsInfo(String diagnosticsInfo);
-
- @Public
- @Unstable
- public abstract FinalApplicationStatus getFinalApplicationStatus();
-
- @Public
- @Unstable
- public abstract void setFinalApplicationStatus(
- FinalApplicationStatus finalApplicationStatus);
-
- @Public
- @Unstable
- public abstract YarnApplicationState getYarnApplicationState();
-
- @Public
- @Unstable
- public abstract void setYarnApplicationState(
- YarnApplicationState yarnApplicationState);
-
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/ApplicationHistoryData.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/ApplicationHistoryData.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/ApplicationHistoryData.java
deleted file mode 100644
index b7d16f3..0000000
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/ApplicationHistoryData.java
+++ /dev/null
@@ -1,213 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.server.applicationhistoryservice.records;
-
-import org.apache.hadoop.classification.InterfaceAudience.Public;
-import org.apache.hadoop.classification.InterfaceStability.Unstable;
-import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
-import org.apache.hadoop.yarn.api.records.YarnApplicationState;
-
-/**
- * The class contains all the fields that are stored persistently for
- * <code>RMApp</code>.
- */
-@Public
-@Unstable
-public class ApplicationHistoryData {
-
- private ApplicationId applicationId;
-
- private String applicationName;
-
- private String applicationType;
-
- private String user;
-
- private String queue;
-
- private long submitTime;
-
- private long startTime;
-
- private long finishTime;
-
- private String diagnosticsInfo;
-
- private FinalApplicationStatus finalApplicationStatus;
-
- private YarnApplicationState yarnApplicationState;
-
- @Public
- @Unstable
- public static ApplicationHistoryData newInstance(ApplicationId applicationId,
- String applicationName, String applicationType, String queue,
- String user, long submitTime, long startTime, long finishTime,
- String diagnosticsInfo, FinalApplicationStatus finalApplicationStatus,
- YarnApplicationState yarnApplicationState) {
- ApplicationHistoryData appHD = new ApplicationHistoryData();
- appHD.setApplicationId(applicationId);
- appHD.setApplicationName(applicationName);
- appHD.setApplicationType(applicationType);
- appHD.setQueue(queue);
- appHD.setUser(user);
- appHD.setSubmitTime(submitTime);
- appHD.setStartTime(startTime);
- appHD.setFinishTime(finishTime);
- appHD.setDiagnosticsInfo(diagnosticsInfo);
- appHD.setFinalApplicationStatus(finalApplicationStatus);
- appHD.setYarnApplicationState(yarnApplicationState);
- return appHD;
- }
-
- @Public
- @Unstable
- public ApplicationId getApplicationId() {
- return applicationId;
- }
-
- @Public
- @Unstable
- public void setApplicationId(ApplicationId applicationId) {
- this.applicationId = applicationId;
- }
-
- @Public
- @Unstable
- public String getApplicationName() {
- return applicationName;
- }
-
- @Public
- @Unstable
- public void setApplicationName(String applicationName) {
- this.applicationName = applicationName;
- }
-
- @Public
- @Unstable
- public String getApplicationType() {
- return applicationType;
- }
-
- @Public
- @Unstable
- public void setApplicationType(String applicationType) {
- this.applicationType = applicationType;
- }
-
- @Public
- @Unstable
- public String getUser() {
- return user;
- }
-
- @Public
- @Unstable
- public void setUser(String user) {
- this.user = user;
- }
-
- @Public
- @Unstable
- public String getQueue() {
- return queue;
- }
-
- @Public
- @Unstable
- public void setQueue(String queue) {
- this.queue = queue;
- }
-
- @Public
- @Unstable
- public long getSubmitTime() {
- return submitTime;
- }
-
- @Public
- @Unstable
- public void setSubmitTime(long submitTime) {
- this.submitTime = submitTime;
- }
-
- @Public
- @Unstable
- public long getStartTime() {
- return startTime;
- }
-
- @Public
- @Unstable
- public void setStartTime(long startTime) {
- this.startTime = startTime;
- }
-
- @Public
- @Unstable
- public long getFinishTime() {
- return finishTime;
- }
-
- @Public
- @Unstable
- public void setFinishTime(long finishTime) {
- this.finishTime = finishTime;
- }
-
- @Public
- @Unstable
- public String getDiagnosticsInfo() {
- return diagnosticsInfo;
- }
-
- @Public
- @Unstable
- public void setDiagnosticsInfo(String diagnosticsInfo) {
- this.diagnosticsInfo = diagnosticsInfo;
- }
-
- @Public
- @Unstable
- public FinalApplicationStatus getFinalApplicationStatus() {
- return finalApplicationStatus;
- }
-
- @Public
- @Unstable
- public void setFinalApplicationStatus(
- FinalApplicationStatus finalApplicationStatus) {
- this.finalApplicationStatus = finalApplicationStatus;
- }
-
- @Public
- @Unstable
- public YarnApplicationState getYarnApplicationState() {
- return this.yarnApplicationState;
- }
-
- @Public
- @Unstable
- public void
- setYarnApplicationState(YarnApplicationState yarnApplicationState) {
- this.yarnApplicationState = yarnApplicationState;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/ApplicationStartData.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/ApplicationStartData.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/ApplicationStartData.java
deleted file mode 100644
index 6bc1323..0000000
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/ApplicationStartData.java
+++ /dev/null
@@ -1,106 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.server.applicationhistoryservice.records;
-
-import org.apache.hadoop.classification.InterfaceAudience.Public;
-import org.apache.hadoop.classification.InterfaceStability.Unstable;
-import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.util.Records;
-
-/**
- * The class contains the fields that can be determined when <code>RMApp</code>
- * starts, and that need to be stored persistently.
- */
-@Public
-@Unstable
-public abstract class ApplicationStartData {
-
- @Public
- @Unstable
- public static ApplicationStartData newInstance(ApplicationId applicationId,
- String applicationName, String applicationType, String queue,
- String user, long submitTime, long startTime) {
- ApplicationStartData appSD = Records.newRecord(ApplicationStartData.class);
- appSD.setApplicationId(applicationId);
- appSD.setApplicationName(applicationName);
- appSD.setApplicationType(applicationType);
- appSD.setQueue(queue);
- appSD.setUser(user);
- appSD.setSubmitTime(submitTime);
- appSD.setStartTime(startTime);
- return appSD;
- }
-
- @Public
- @Unstable
- public abstract ApplicationId getApplicationId();
-
- @Public
- @Unstable
- public abstract void setApplicationId(ApplicationId applicationId);
-
- @Public
- @Unstable
- public abstract String getApplicationName();
-
- @Public
- @Unstable
- public abstract void setApplicationName(String applicationName);
-
- @Public
- @Unstable
- public abstract String getApplicationType();
-
- @Public
- @Unstable
- public abstract void setApplicationType(String applicationType);
-
- @Public
- @Unstable
- public abstract String getUser();
-
- @Public
- @Unstable
- public abstract void setUser(String user);
-
- @Public
- @Unstable
- public abstract String getQueue();
-
- @Public
- @Unstable
- public abstract void setQueue(String queue);
-
- @Public
- @Unstable
- public abstract long getSubmitTime();
-
- @Public
- @Unstable
- public abstract void setSubmitTime(long submitTime);
-
- @Public
- @Unstable
- public abstract long getStartTime();
-
- @Public
- @Unstable
- public abstract void setStartTime(long startTime);
-
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/ContainerFinishData.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/ContainerFinishData.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/ContainerFinishData.java
deleted file mode 100644
index 5eb9ddb..0000000
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/ContainerFinishData.java
+++ /dev/null
@@ -1,90 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.server.applicationhistoryservice.records;
-
-import org.apache.hadoop.classification.InterfaceAudience.Public;
-import org.apache.hadoop.classification.InterfaceStability.Unstable;
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.api.records.ContainerState;
-import org.apache.hadoop.yarn.util.Records;
-
-/**
- * The class contains the fields that can be determined when
- * <code>RMContainer</code> finishes, and that need to be stored persistently.
- */
-@Public
-@Unstable
-public abstract class ContainerFinishData {
-
- @Public
- @Unstable
- public static ContainerFinishData newInstance(ContainerId containerId,
- long finishTime, String diagnosticsInfo, int containerExitCode,
- ContainerState containerState) {
- ContainerFinishData containerFD =
- Records.newRecord(ContainerFinishData.class);
- containerFD.setContainerId(containerId);
- containerFD.setFinishTime(finishTime);
- containerFD.setDiagnosticsInfo(diagnosticsInfo);
- containerFD.setContainerExitStatus(containerExitCode);
- containerFD.setContainerState(containerState);
- return containerFD;
- }
-
- @Public
- @Unstable
- public abstract ContainerId getContainerId();
-
- @Public
- @Unstable
- public abstract void setContainerId(ContainerId containerId);
-
- @Public
- @Unstable
- public abstract long getFinishTime();
-
- @Public
- @Unstable
- public abstract void setFinishTime(long finishTime);
-
- @Public
- @Unstable
- public abstract String getDiagnosticsInfo();
-
- @Public
- @Unstable
- public abstract void setDiagnosticsInfo(String diagnosticsInfo);
-
- @Public
- @Unstable
- public abstract int getContainerExitStatus();
-
- @Public
- @Unstable
- public abstract void setContainerExitStatus(int containerExitStatus);
-
- @Public
- @Unstable
- public abstract ContainerState getContainerState();
-
- @Public
- @Unstable
- public abstract void setContainerState(ContainerState containerState);
-
-}
[17/28] ambari git commit: AMBARI-22740 : Fix integration test for
HBase in branch-3.0-ams due to UUID changes. (avijayan)
Posted by av...@apache.org.
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/PhoenixHBaseAccessor.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/PhoenixHBaseAccessor.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/PhoenixHBaseAccessor.java
deleted file mode 100644
index c438299..0000000
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/PhoenixHBaseAccessor.java
+++ /dev/null
@@ -1,1904 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline;
-
-import static java.util.concurrent.TimeUnit.SECONDS;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.AGGREGATORS_SKIP_BLOCK_CACHE;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.CLUSTER_DAILY_TABLE_TTL;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.CLUSTER_HOUR_TABLE_TTL;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.CLUSTER_MINUTE_TABLE_TTL;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.CLUSTER_SECOND_TABLE_TTL;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.CONTAINER_METRICS_TTL;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.GLOBAL_MAX_RETRIES;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.GLOBAL_RESULT_LIMIT;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.GLOBAL_RETRY_INTERVAL;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.HBASE_BLOCKING_STORE_FILES;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.HBASE_COMPRESSION_SCHEME;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.HBASE_ENCODING_SCHEME;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.HOST_DAILY_TABLE_TTL;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.HOST_HOUR_TABLE_TTL;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.HOST_MINUTE_TABLE_TTL;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.OUT_OFF_BAND_DATA_TIME_ALLOWANCE;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.PRECISION_TABLE_TTL;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.TIMELINE_METRICS_AGGREGATE_TABLES_DURABILITY;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.TIMELINE_METRICS_AGGREGATE_TABLE_HBASE_BLOCKING_STORE_FILES;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.TIMELINE_METRICS_CACHE_COMMIT_INTERVAL;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.TIMELINE_METRICS_CACHE_ENABLED;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.TIMELINE_METRICS_CACHE_SIZE;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.TIMELINE_METRICS_HBASE_AGGREGATE_TABLE_COMPACTION_POLICY_CLASS;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.TIMELINE_METRICS_HBASE_AGGREGATE_TABLE_COMPACTION_POLICY_KEY;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.TIMELINE_METRICS_HBASE_PRECISION_TABLE_COMPACTION_POLICY_CLASS;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.TIMELINE_METRICS_HBASE_PRECISION_TABLE_COMPACTION_POLICY_KEY;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.TIMELINE_METRICS_PRECISION_TABLE_DURABILITY;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.TIMELINE_METRICS_PRECISION_TABLE_HBASE_BLOCKING_STORE_FILES;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.TIMELINE_METRIC_AGGREGATOR_SINK_CLASS;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query.PhoenixTransactSQL.ALTER_METRICS_METADATA_TABLE;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query.PhoenixTransactSQL.CONTAINER_METRICS_TABLE_NAME;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query.PhoenixTransactSQL.CREATE_CONTAINER_METRICS_TABLE_SQL;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query.PhoenixTransactSQL.CREATE_HOSTED_APPS_METADATA_TABLE_SQL;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query.PhoenixTransactSQL.CREATE_INSTANCE_HOST_TABLE_SQL;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query.PhoenixTransactSQL.CREATE_METRICS_AGGREGATE_TABLE_SQL;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query.PhoenixTransactSQL.CREATE_METRICS_CLUSTER_AGGREGATE_GROUPED_TABLE_SQL;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query.PhoenixTransactSQL.CREATE_METRICS_CLUSTER_AGGREGATE_TABLE_SQL;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query.PhoenixTransactSQL.CREATE_METRICS_METADATA_TABLE_SQL;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query.PhoenixTransactSQL.CREATE_METRICS_TABLE_SQL;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query.PhoenixTransactSQL.DEFAULT_ENCODING;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query.PhoenixTransactSQL.DEFAULT_TABLE_COMPRESSION;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query.PhoenixTransactSQL.GET_HOSTED_APPS_METADATA_SQL;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query.PhoenixTransactSQL.GET_INSTANCE_HOST_METADATA_SQL;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query.PhoenixTransactSQL.GET_METRIC_METADATA_SQL;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query.PhoenixTransactSQL.METRICS_AGGREGATE_DAILY_TABLE_NAME;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query.PhoenixTransactSQL.METRICS_AGGREGATE_HOURLY_TABLE_NAME;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query.PhoenixTransactSQL.METRICS_AGGREGATE_MINUTE_TABLE_NAME;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query.PhoenixTransactSQL.METRICS_CLUSTER_AGGREGATE_DAILY_TABLE_NAME;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query.PhoenixTransactSQL.METRICS_CLUSTER_AGGREGATE_HOURLY_TABLE_NAME;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query.PhoenixTransactSQL.METRICS_CLUSTER_AGGREGATE_MINUTE_TABLE_NAME;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query.PhoenixTransactSQL.METRICS_CLUSTER_AGGREGATE_TABLE_NAME;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query.PhoenixTransactSQL.METRICS_RECORD_TABLE_NAME;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query.PhoenixTransactSQL.PHOENIX_TABLES;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query.PhoenixTransactSQL.PHOENIX_TABLES_REGEX_PATTERN;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query.PhoenixTransactSQL.UPSERT_AGGREGATE_RECORD_SQL;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query.PhoenixTransactSQL.UPSERT_CLUSTER_AGGREGATE_SQL;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query.PhoenixTransactSQL.UPSERT_CLUSTER_AGGREGATE_TIME_SQL;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query.PhoenixTransactSQL.UPSERT_CONTAINER_METRICS_SQL;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query.PhoenixTransactSQL.UPSERT_HOSTED_APPS_METADATA_SQL;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query.PhoenixTransactSQL.UPSERT_INSTANCE_HOST_METADATA_SQL;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query.PhoenixTransactSQL.UPSERT_METADATA_SQL;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query.PhoenixTransactSQL.UPSERT_METRICS_SQL;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.source.InternalSourceProvider.SOURCE_NAME.RAW_METRICS;
-
-import java.io.IOException;
-import java.sql.Connection;
-import java.sql.PreparedStatement;
-import java.sql.ResultSet;
-import java.sql.SQLException;
-import java.sql.Statement;
-import java.sql.Timestamp;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.Optional;
-import java.util.Set;
-import java.util.TreeMap;
-import java.util.concurrent.ArrayBlockingQueue;
-import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.Executors;
-import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.TimeUnit;
-
-import org.apache.commons.lang.StringUtils;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.DoNotRetryIOException;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.Admin;
-import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
-import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
-import org.apache.hadoop.hbase.client.Durability;
-import org.apache.hadoop.hbase.client.TableDescriptor;
-import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
-import org.apache.hadoop.hbase.util.RetryCounter;
-import org.apache.hadoop.hbase.util.RetryCounterFactory;
-import org.apache.hadoop.metrics2.sink.timeline.ContainerMetric;
-import org.apache.hadoop.metrics2.sink.timeline.MetricClusterAggregate;
-import org.apache.hadoop.metrics2.sink.timeline.MetricHostAggregate;
-import org.apache.hadoop.metrics2.sink.timeline.Precision;
-import org.apache.hadoop.metrics2.sink.timeline.SingleValuedTimelineMetric;
-import org.apache.hadoop.metrics2.sink.timeline.TimelineMetric;
-import org.apache.hadoop.metrics2.sink.timeline.TimelineMetricMetadata;
-import org.apache.hadoop.metrics2.sink.timeline.TimelineMetrics;
-import org.apache.hadoop.util.ReflectionUtils;
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.aggregators.AggregatorUtils;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.aggregators.Function;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.aggregators.TimelineClusterMetric;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.aggregators.TimelineMetricReadHelper;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.discovery.TimelineMetricHostMetadata;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.discovery.TimelineMetricMetadataKey;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.discovery.TimelineMetricMetadataManager;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query.Condition;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query.DefaultPhoenixDataSource;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query.PhoenixConnectionProvider;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query.PhoenixTransactSQL;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query.SplitByMetricNamesCondition;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.sink.ExternalMetricsSink;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.sink.ExternalSinkProvider;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.source.InternalMetricsSource;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.source.InternalSourceProvider;
-import org.apache.hadoop.yarn.util.timeline.TimelineUtils;
-import org.apache.phoenix.exception.PhoenixIOException;
-import org.codehaus.jackson.map.ObjectMapper;
-import org.codehaus.jackson.type.TypeReference;
-
-import com.google.common.collect.Multimap;
-
-
-/**
- * Provides a facade over the Phoenix API to access HBase schema
- */
-public class PhoenixHBaseAccessor {
- private static final Log LOG = LogFactory.getLog(PhoenixHBaseAccessor.class);
-
- static final int PHOENIX_MAX_MUTATION_STATE_SIZE = 50000;
- // Default stale data allowance set to 3 minutes, 2 minutes more than time
- // it was collected. Also 2 minutes is the default aggregation interval at
- // cluster and host levels.
- static final long DEFAULT_OUT_OF_BAND_TIME_ALLOWANCE = 300000;
- /**
- * 22 metrics for 2hours in SECONDS (10 second data)
- * => Reasonable upper bound on the limit such that our Precision calculation for a given time range makes sense.
- */
- private static final int METRICS_PER_MINUTE = 22;
- private static final int POINTS_PER_MINUTE = 6;
- public static int RESULTSET_LIMIT = (int)TimeUnit.HOURS.toMinutes(2) * METRICS_PER_MINUTE * POINTS_PER_MINUTE ;
-
- static TimelineMetricReadHelper TIMELINE_METRIC_READ_HELPER = new TimelineMetricReadHelper();
- static ObjectMapper mapper = new ObjectMapper();
- static TypeReference<TreeMap<Long, Double>> metricValuesTypeRef = new TypeReference<TreeMap<Long, Double>>() {};
-
- private final Configuration hbaseConf;
- private final Configuration metricsConf;
- private final RetryCounterFactory retryCounterFactory;
- private final PhoenixConnectionProvider dataSource;
- private final long outOfBandTimeAllowance;
- private final int cacheSize;
- private final boolean cacheEnabled;
- private final BlockingQueue<TimelineMetrics> insertCache;
- private ScheduledExecutorService scheduledExecutorService;
- private MetricsCacheCommitterThread metricsCommiterThread;
- private TimelineMetricsAggregatorSink aggregatorSink;
- private final int cacheCommitInterval;
- private final boolean skipBlockCacheForAggregatorsEnabled;
- private final String timelineMetricsTablesDurability;
- private final String timelineMetricsPrecisionTableDurability;
- private TimelineMetricMetadataManager metadataManagerInstance;
-
- static final String HSTORE_COMPACTION_CLASS_KEY =
- "hbase.hstore.defaultengine.compactionpolicy.class";
- static final String HSTORE_ENGINE_CLASS =
- "hbase.hstore.engine.class";
- static final String FIFO_COMPACTION_POLICY_CLASS =
- "org.apache.hadoop.hbase.regionserver.compactions.FIFOCompactionPolicy";
- static final String DATE_TIERED_COMPACTION_POLICY =
- "org.apache.hadoop.hbase.regionserver.DateTieredStoreEngine";
- static final String BLOCKING_STORE_FILES_KEY =
- "hbase.hstore.blockingStoreFiles";
-
- private Map<String, Integer> tableTTL = new HashMap<>();
-
- private final TimelineMetricConfiguration configuration;
- private List<InternalMetricsSource> rawMetricsSources = new ArrayList<>();
-
- public PhoenixHBaseAccessor(PhoenixConnectionProvider dataSource) {
- this(TimelineMetricConfiguration.getInstance(), dataSource);
- }
-
- // Test friendly construction since mock instrumentation is difficult to get
- // working with hadoop mini cluster
- PhoenixHBaseAccessor(TimelineMetricConfiguration configuration,
- PhoenixConnectionProvider dataSource) {
- this.configuration = TimelineMetricConfiguration.getInstance();
- try {
- this.hbaseConf = configuration.getHbaseConf();
- this.metricsConf = configuration.getMetricsConf();
- } catch (Exception e) {
- throw new ExceptionInInitializerError("Cannot initialize configuration.");
- }
- if (dataSource == null) {
- dataSource = new DefaultPhoenixDataSource(hbaseConf);
- }
- this.dataSource = dataSource;
-
- RESULTSET_LIMIT = metricsConf.getInt(GLOBAL_RESULT_LIMIT, RESULTSET_LIMIT);
- try {
- Class.forName("org.apache.phoenix.jdbc.PhoenixDriver");
- } catch (ClassNotFoundException e) {
- LOG.error("Phoenix client jar not found in the classpath.", e);
- throw new IllegalStateException(e);
- }
-
- this.retryCounterFactory = new RetryCounterFactory(metricsConf.getInt(GLOBAL_MAX_RETRIES, 10),
- (int) SECONDS.toMillis(metricsConf.getInt(GLOBAL_RETRY_INTERVAL, 3)));
- this.outOfBandTimeAllowance = metricsConf.getLong(OUT_OFF_BAND_DATA_TIME_ALLOWANCE,
- DEFAULT_OUT_OF_BAND_TIME_ALLOWANCE);
- this.cacheEnabled = Boolean.valueOf(metricsConf.get(TIMELINE_METRICS_CACHE_ENABLED, "true"));
- this.cacheSize = Integer.valueOf(metricsConf.get(TIMELINE_METRICS_CACHE_SIZE, "150"));
- this.cacheCommitInterval = Integer.valueOf(metricsConf.get(TIMELINE_METRICS_CACHE_COMMIT_INTERVAL, "3"));
- this.insertCache = new ArrayBlockingQueue<TimelineMetrics>(cacheSize);
- this.skipBlockCacheForAggregatorsEnabled = metricsConf.getBoolean(AGGREGATORS_SKIP_BLOCK_CACHE, false);
- this.timelineMetricsTablesDurability = metricsConf.get(TIMELINE_METRICS_AGGREGATE_TABLES_DURABILITY, "");
- this.timelineMetricsPrecisionTableDurability = metricsConf.get(TIMELINE_METRICS_PRECISION_TABLE_DURABILITY, "");
-
- tableTTL.put(METRICS_RECORD_TABLE_NAME, metricsConf.getInt(PRECISION_TABLE_TTL, 1 * 86400)); // 1 day
- tableTTL.put(CONTAINER_METRICS_TABLE_NAME, metricsConf.getInt(CONTAINER_METRICS_TTL, 30 * 86400)); // 30 days
- tableTTL.put(METRICS_AGGREGATE_MINUTE_TABLE_NAME, metricsConf.getInt(HOST_MINUTE_TABLE_TTL, 7 * 86400)); //7 days
- tableTTL.put(METRICS_AGGREGATE_HOURLY_TABLE_NAME, metricsConf.getInt(HOST_HOUR_TABLE_TTL, 30 * 86400)); //30 days
- tableTTL.put(METRICS_AGGREGATE_DAILY_TABLE_NAME, metricsConf.getInt(HOST_DAILY_TABLE_TTL, 365 * 86400)); //1 year
- tableTTL.put(METRICS_CLUSTER_AGGREGATE_TABLE_NAME, metricsConf.getInt(CLUSTER_SECOND_TABLE_TTL, 7 * 86400)); //7 days
- tableTTL.put(METRICS_CLUSTER_AGGREGATE_MINUTE_TABLE_NAME, metricsConf.getInt(CLUSTER_MINUTE_TABLE_TTL, 30 * 86400)); //30 days
- tableTTL.put(METRICS_CLUSTER_AGGREGATE_HOURLY_TABLE_NAME, metricsConf.getInt(CLUSTER_HOUR_TABLE_TTL, 365 * 86400)); //1 year
- tableTTL.put(METRICS_CLUSTER_AGGREGATE_DAILY_TABLE_NAME, metricsConf.getInt(CLUSTER_DAILY_TABLE_TTL, 730 * 86400)); //2 years
-
- if (cacheEnabled) {
- LOG.debug("Initialising and starting metrics cache committer thread...");
- metricsCommiterThread = new MetricsCacheCommitterThread(this);
- scheduledExecutorService = Executors.newSingleThreadScheduledExecutor();
- scheduledExecutorService.scheduleWithFixedDelay(metricsCommiterThread, 0, cacheCommitInterval, TimeUnit.SECONDS);
- }
-
- Class<? extends TimelineMetricsAggregatorSink> metricSinkClass =
- metricsConf.getClass(TIMELINE_METRIC_AGGREGATOR_SINK_CLASS, null,
- TimelineMetricsAggregatorSink.class);
- if (metricSinkClass != null) {
- aggregatorSink = ReflectionUtils.newInstance(metricSinkClass, metricsConf);
- LOG.info("Initialized aggregator sink class " + metricSinkClass);
- }
-
- List<ExternalSinkProvider> externalSinkProviderList = configuration.getExternalSinkProviderList();
- InternalSourceProvider internalSourceProvider = configuration.getInternalSourceProvider();
- if (!externalSinkProviderList.isEmpty()) {
- for (ExternalSinkProvider externalSinkProvider : externalSinkProviderList) {
- ExternalMetricsSink rawMetricsSink = externalSinkProvider.getExternalMetricsSink(RAW_METRICS);
- int interval = configuration.getExternalSinkInterval(externalSinkProvider.getClass().getSimpleName(), RAW_METRICS);
- if (interval == -1) {
- interval = cacheCommitInterval;
- }
- rawMetricsSources.add(internalSourceProvider.getInternalMetricsSource(RAW_METRICS, interval, rawMetricsSink));
- }
- }
- TIMELINE_METRIC_READ_HELPER = new TimelineMetricReadHelper(this.metadataManagerInstance);
- }
-
- public boolean isInsertCacheEmpty() {
- return insertCache.isEmpty();
- }
-
- public void commitMetricsFromCache() {
- LOG.debug("Clearing metrics cache");
- List<TimelineMetrics> metricsList = new ArrayList<TimelineMetrics>(insertCache.size());
- if (!insertCache.isEmpty()) {
- insertCache.drainTo(metricsList); // More performant than poll
- }
- if (metricsList.size() > 0) {
- commitMetrics(metricsList);
- if (!rawMetricsSources.isEmpty()) {
- for (InternalMetricsSource rawMetricsSource : rawMetricsSources) {
- rawMetricsSource.publishTimelineMetrics(metricsList);
- }
- }
- }
- }
-
- public void commitMetrics(TimelineMetrics timelineMetrics) {
- commitMetrics(Collections.singletonList(timelineMetrics));
- }
-
- public void commitMetrics(Collection<TimelineMetrics> timelineMetricsCollection) {
- LOG.debug("Committing metrics to store");
- Connection conn = null;
- PreparedStatement metricRecordStmt = null;
-
- try {
- conn = getConnection();
- metricRecordStmt = conn.prepareStatement(String.format(
- UPSERT_METRICS_SQL, METRICS_RECORD_TABLE_NAME));
- for (TimelineMetrics timelineMetrics : timelineMetricsCollection) {
- for (TimelineMetric metric : timelineMetrics.getMetrics()) {
-
- metricRecordStmt.clearParameters();
-
- if (LOG.isTraceEnabled()) {
- LOG.trace("host: " + metric.getHostName() + ", " +
- "metricName = " + metric.getMetricName() + ", " +
- "values: " + metric.getMetricValues());
- }
- double[] aggregates = AggregatorUtils.calculateAggregates(
- metric.getMetricValues());
-
- byte[] uuid = metadataManagerInstance.getUuid(metric);
- if (uuid == null) {
- LOG.error("Error computing UUID for metric. Cannot write metrics : " + metric.toString());
- continue;
- }
- metricRecordStmt.setBytes(1, uuid);
- metricRecordStmt.setLong(2, metric.getStartTime());
- metricRecordStmt.setDouble(3, aggregates[0]);
- metricRecordStmt.setDouble(4, aggregates[1]);
- metricRecordStmt.setDouble(5, aggregates[2]);
- metricRecordStmt.setLong(6, (long) aggregates[3]);
- String json = TimelineUtils.dumpTimelineRecordtoJSON(metric.getMetricValues());
- metricRecordStmt.setString(7, json);
-
- try {
- metricRecordStmt.executeUpdate();
- } catch (SQLException sql) {
- LOG.error("Failed on insert records to store.", sql);
- }
- }
- }
-
- // commit() blocked if HBase unavailable
- conn.commit();
- } catch (Exception exception){
- exception.printStackTrace();
- }
- finally {
- if (metricRecordStmt != null) {
- try {
- metricRecordStmt.close();
- } catch (SQLException e) {
- // Ignore
- }
- }
- if (conn != null) {
- try {
- conn.close();
- } catch (SQLException sql) {
- // Ignore
- }
- }
- }
- }
-
- private static TimelineMetric getLastTimelineMetricFromResultSet(ResultSet rs)
- throws SQLException, IOException {
- TimelineMetric metric = TIMELINE_METRIC_READ_HELPER.getTimelineMetricCommonsFromResultSet(rs);
- metric.setMetricValues(readLastMetricValueFromJSON(rs.getString("METRICS")));
- return metric;
- }
-
- private static TreeMap<Long, Double> readLastMetricValueFromJSON(String json)
- throws IOException {
- TreeMap<Long, Double> values = readMetricFromJSON(json);
- Long lastTimeStamp = values.lastKey();
-
- TreeMap<Long, Double> valueMap = new TreeMap<Long, Double>();
- valueMap.put(lastTimeStamp, values.get(lastTimeStamp));
- return valueMap;
- }
-
- @SuppressWarnings("unchecked")
- public static TreeMap<Long, Double> readMetricFromJSON(String json) throws IOException {
- return mapper.readValue(json, metricValuesTypeRef);
- }
-
- private Connection getConnectionRetryingOnException()
- throws SQLException, InterruptedException {
- RetryCounter retryCounter = retryCounterFactory.create();
- while (true) {
- try{
- return getConnection();
- } catch (SQLException e) {
- if(!retryCounter.shouldRetry()){
- LOG.error("HBaseAccessor getConnection failed after "
- + retryCounter.getMaxAttempts() + " attempts");
- throw e;
- }
- }
- retryCounter.sleepUntilNextRetry();
- }
- }
-
- /**
- * Get JDBC connection to HBase store. Assumption is that the hbase
- * configuration is present on the classpath and loaded by the caller into
- * the Configuration object.
- * Phoenix already caches the HConnection between the client and HBase
- * cluster.
- *
- * @return @java.sql.Connection
- */
- public Connection getConnection() throws SQLException {
- return dataSource.getConnection();
- }
-
- /**
- * Unit test purpose only for now.
- * @return @HBaseAdmin
- * @throws IOException
- */
- Admin getHBaseAdmin() throws IOException {
- return dataSource.getHBaseAdmin();
- }
-
- protected void initMetricSchema() {
- Connection conn = null;
- Statement stmt = null;
-
- String encoding = metricsConf.get(HBASE_ENCODING_SCHEME, DEFAULT_ENCODING);
- String compression = metricsConf.get(HBASE_COMPRESSION_SCHEME, DEFAULT_TABLE_COMPRESSION);
-
-
- try {
- LOG.info("Initializing metrics schema...");
- conn = getConnectionRetryingOnException();
- stmt = conn.createStatement();
-
- // Metadata
- String metadataSql = String.format(CREATE_METRICS_METADATA_TABLE_SQL,
- encoding, compression);
- stmt.executeUpdate(metadataSql);
- stmt.executeUpdate(ALTER_METRICS_METADATA_TABLE);
-
- String hostedAppSql = String.format(CREATE_HOSTED_APPS_METADATA_TABLE_SQL,
- encoding, compression);
- stmt.executeUpdate(hostedAppSql);
-
- //Host Instances table
- String hostedInstancesSql = String.format(CREATE_INSTANCE_HOST_TABLE_SQL,
- encoding, compression);
- stmt.executeUpdate(hostedInstancesSql);
-
- // Container Metrics
- stmt.executeUpdate( String.format(CREATE_CONTAINER_METRICS_TABLE_SQL,
- encoding, tableTTL.get(CONTAINER_METRICS_TABLE_NAME), compression));
-
- // Host level
- String precisionSql = String.format(CREATE_METRICS_TABLE_SQL,
- encoding, tableTTL.get(METRICS_RECORD_TABLE_NAME), compression);
- stmt.executeUpdate(precisionSql);
-
- String hostMinuteAggregrateSql = String.format(CREATE_METRICS_AGGREGATE_TABLE_SQL,
- METRICS_AGGREGATE_MINUTE_TABLE_NAME, encoding,
- tableTTL.get(METRICS_AGGREGATE_MINUTE_TABLE_NAME),
- compression);
- stmt.executeUpdate(hostMinuteAggregrateSql);
-
- stmt.executeUpdate(String.format(CREATE_METRICS_AGGREGATE_TABLE_SQL,
- METRICS_AGGREGATE_HOURLY_TABLE_NAME, encoding,
- tableTTL.get(METRICS_AGGREGATE_HOURLY_TABLE_NAME),
- compression));
- stmt.executeUpdate(String.format(CREATE_METRICS_AGGREGATE_TABLE_SQL,
- METRICS_AGGREGATE_DAILY_TABLE_NAME, encoding,
- tableTTL.get(METRICS_AGGREGATE_DAILY_TABLE_NAME),
- compression));
-
- // Cluster level
- String aggregateSql = String.format(CREATE_METRICS_CLUSTER_AGGREGATE_TABLE_SQL,
- METRICS_CLUSTER_AGGREGATE_TABLE_NAME, encoding,
- tableTTL.get(METRICS_CLUSTER_AGGREGATE_TABLE_NAME),
- compression);
-
- stmt.executeUpdate(aggregateSql);
- stmt.executeUpdate(String.format(CREATE_METRICS_CLUSTER_AGGREGATE_GROUPED_TABLE_SQL,
- METRICS_CLUSTER_AGGREGATE_MINUTE_TABLE_NAME, encoding,
- tableTTL.get(METRICS_CLUSTER_AGGREGATE_MINUTE_TABLE_NAME),
- compression));
- stmt.executeUpdate(String.format(CREATE_METRICS_CLUSTER_AGGREGATE_GROUPED_TABLE_SQL,
- METRICS_CLUSTER_AGGREGATE_HOURLY_TABLE_NAME, encoding,
- tableTTL.get(METRICS_CLUSTER_AGGREGATE_HOURLY_TABLE_NAME),
- compression));
- stmt.executeUpdate(String.format(CREATE_METRICS_CLUSTER_AGGREGATE_GROUPED_TABLE_SQL,
- METRICS_CLUSTER_AGGREGATE_DAILY_TABLE_NAME, encoding,
- tableTTL.get(METRICS_CLUSTER_AGGREGATE_DAILY_TABLE_NAME),
- compression));
-
-
- conn.commit();
-
- LOG.info("Metrics schema initialized.");
- } catch (SQLException | InterruptedException sql) {
- LOG.error("Error creating Metrics Schema in HBase using Phoenix.", sql);
- throw new MetricsSystemInitializationException(
- "Error creating Metrics Schema in HBase using Phoenix.", sql);
- } finally {
- if (stmt != null) {
- try {
- stmt.close();
- } catch (SQLException e) {
- // Ignore
- }
- }
- if (conn != null) {
- try {
- conn.close();
- } catch (SQLException e) {
- // Ignore
- }
- }
- }
- }
-
- protected void initPoliciesAndTTL() {
- Admin hBaseAdmin = null;
- try {
- hBaseAdmin = dataSource.getHBaseAdmin();
- } catch (IOException e) {
- LOG.warn("Unable to initialize HBaseAdmin for setting policies.", e);
- }
-
- TableName[] tableNames = null;
- if (hBaseAdmin != null) {
- try {
- tableNames = hBaseAdmin.listTableNames(PHOENIX_TABLES_REGEX_PATTERN, false);
- } catch (IOException e) {
- LOG.warn("Unable to get table names from HBaseAdmin for setting policies.", e);
- return;
- }
- if (tableNames == null || tableNames.length == 0) {
- LOG.warn("Unable to get table names from HBaseAdmin for setting policies.");
- return;
- }
- for (String tableName : PHOENIX_TABLES) {
- try {
- boolean modifyTable = false;
- Optional<TableName> tableNameOptional = Arrays.stream(tableNames)
- .filter(t -> tableName.equals(t.getNameAsString())).findFirst();
-
- TableDescriptor tableDescriptor = null;
- if (tableNameOptional.isPresent()) {
- tableDescriptor = hBaseAdmin.getTableDescriptor(tableNameOptional.get());
- }
-
- if (tableDescriptor == null) {
- LOG.warn("Unable to get table descriptor for " + tableName);
- continue;
- }
-
- // @TableDescriptor is immutable by design
- TableDescriptorBuilder tableDescriptorBuilder =
- TableDescriptorBuilder.newBuilder(tableDescriptor);
-
- //Set normalizer preferences
- boolean enableNormalizer = hbaseConf.getBoolean("hbase.normalizer.enabled", false);
- if (enableNormalizer ^ tableDescriptor.isNormalizationEnabled()) {
- tableDescriptorBuilder.setNormalizationEnabled(enableNormalizer);
- LOG.info("Normalizer set to " + enableNormalizer + " for " + tableName);
- modifyTable = true;
- }
-
- //Set durability preferences
- boolean durabilitySettingsModified = setDurabilityForTable(tableName, tableDescriptorBuilder);
- modifyTable = modifyTable || durabilitySettingsModified;
-
- //Set compaction policy preferences
- boolean compactionPolicyModified = false;
- compactionPolicyModified = setCompactionPolicyForTable(tableName, tableDescriptorBuilder);
- modifyTable = modifyTable || compactionPolicyModified;
-
- // Change TTL setting to match user configuration
- ColumnFamilyDescriptor[] columnFamilyDescriptors = tableDescriptor.getColumnFamilies();
- if (columnFamilyDescriptors != null) {
- for (ColumnFamilyDescriptor familyDescriptor : columnFamilyDescriptors) {
- int ttlValue = familyDescriptor.getTimeToLive();
- if (ttlValue != tableTTL.get(tableName)) {
- ColumnFamilyDescriptorBuilder familyDescriptorBuilder =
- ColumnFamilyDescriptorBuilder.newBuilder(familyDescriptor);
-
- familyDescriptorBuilder.setTimeToLive(tableTTL.get(tableName));
-
- LOG.info("Setting TTL on table: " + tableName + " to : " +
- tableTTL.get(tableName) + " seconds.");
-
- hBaseAdmin.modifyColumnFamily(tableNameOptional.get(), familyDescriptorBuilder.build());
- // modifyTable = true;
- }
- }
- }
-
- // Persist only if anything changed
- if (modifyTable) {
- hBaseAdmin.modifyTable(tableNameOptional.get(), tableDescriptorBuilder.build());
- }
-
- } catch (IOException e) {
- LOG.error("Failed setting policies for " + tableName, e);
- }
- }
- try {
- hBaseAdmin.close();
- } catch (IOException e) {
- LOG.warn("Exception on HBaseAdmin close.", e);
- }
- }
- }
-
- private boolean setDurabilityForTable(String tableName, TableDescriptorBuilder tableDescriptor) {
-
- boolean modifyTable = false;
- // Set WAL preferences
- if (METRICS_RECORD_TABLE_NAME.equals(tableName)) {
- if (!timelineMetricsPrecisionTableDurability.isEmpty()) {
- LOG.info("Setting WAL option " + timelineMetricsPrecisionTableDurability + " for table : " + tableName);
- boolean validDurability = true;
- if ("SKIP_WAL".equals(timelineMetricsPrecisionTableDurability)) {
- tableDescriptor.setDurability(Durability.SKIP_WAL);
- } else if ("SYNC_WAL".equals(timelineMetricsPrecisionTableDurability)) {
- tableDescriptor.setDurability(Durability.SYNC_WAL);
- } else if ("ASYNC_WAL".equals(timelineMetricsPrecisionTableDurability)) {
- tableDescriptor.setDurability(Durability.ASYNC_WAL);
- } else if ("FSYNC_WAL".equals(timelineMetricsPrecisionTableDurability)) {
- tableDescriptor.setDurability(Durability.FSYNC_WAL);
- } else {
- LOG.info("Unknown value for " + TIMELINE_METRICS_PRECISION_TABLE_DURABILITY + " : " + timelineMetricsPrecisionTableDurability);
- validDurability = false;
- }
- if (validDurability) {
- modifyTable = true;
- }
- }
- } else {
- if (!timelineMetricsTablesDurability.isEmpty()) {
- LOG.info("Setting WAL option " + timelineMetricsTablesDurability + " for table : " + tableName);
- boolean validDurability = true;
- if ("SKIP_WAL".equals(timelineMetricsTablesDurability)) {
- tableDescriptor.setDurability(Durability.SKIP_WAL);
- } else if ("SYNC_WAL".equals(timelineMetricsTablesDurability)) {
- tableDescriptor.setDurability(Durability.SYNC_WAL);
- } else if ("ASYNC_WAL".equals(timelineMetricsTablesDurability)) {
- tableDescriptor.setDurability(Durability.ASYNC_WAL);
- } else if ("FSYNC_WAL".equals(timelineMetricsTablesDurability)) {
- tableDescriptor.setDurability(Durability.FSYNC_WAL);
- } else {
- LOG.info("Unknown value for " + TIMELINE_METRICS_AGGREGATE_TABLES_DURABILITY + " : " + timelineMetricsTablesDurability);
- validDurability = false;
- }
- if (validDurability) {
- modifyTable = true;
- }
- }
- }
- return modifyTable;
- }
-
- private boolean setCompactionPolicyForTable(String tableName, TableDescriptorBuilder tableDescriptorBuilder) {
-
- boolean modifyTable = false;
-
- String compactionPolicyKey = metricsConf.get(TIMELINE_METRICS_HBASE_AGGREGATE_TABLE_COMPACTION_POLICY_KEY,
- HSTORE_ENGINE_CLASS);
- String compactionPolicyClass = metricsConf.get(TIMELINE_METRICS_HBASE_AGGREGATE_TABLE_COMPACTION_POLICY_CLASS,
- DATE_TIERED_COMPACTION_POLICY);
- int blockingStoreFiles = hbaseConf.getInt(TIMELINE_METRICS_AGGREGATE_TABLE_HBASE_BLOCKING_STORE_FILES, 60);
-
- if (tableName.equals(METRICS_RECORD_TABLE_NAME)) {
- compactionPolicyKey = metricsConf.get(TIMELINE_METRICS_HBASE_PRECISION_TABLE_COMPACTION_POLICY_KEY,
- HSTORE_COMPACTION_CLASS_KEY);
- compactionPolicyClass = metricsConf.get(TIMELINE_METRICS_HBASE_PRECISION_TABLE_COMPACTION_POLICY_CLASS,
- FIFO_COMPACTION_POLICY_CLASS);
- blockingStoreFiles = hbaseConf.getInt(TIMELINE_METRICS_PRECISION_TABLE_HBASE_BLOCKING_STORE_FILES, 1000);
- }
-
- if (StringUtils.isEmpty(compactionPolicyKey) || StringUtils.isEmpty(compactionPolicyClass)) {
- // Default blockingStoreFiles = 300
- modifyTable = setHbaseBlockingStoreFiles(tableDescriptorBuilder, tableName, 300);
- } else {
- tableDescriptorBuilder.setValue(compactionPolicyKey, compactionPolicyClass);
- tableDescriptorBuilder.removeValue(HSTORE_ENGINE_CLASS.getBytes());
- tableDescriptorBuilder.removeValue(HSTORE_COMPACTION_CLASS_KEY.getBytes());
- setHbaseBlockingStoreFiles(tableDescriptorBuilder, tableName, blockingStoreFiles);
- modifyTable = true;
- }
-
- return modifyTable;
- }
-
- private boolean setHbaseBlockingStoreFiles(TableDescriptorBuilder tableDescriptor,
- String tableName, int value) {
- int blockingStoreFiles = hbaseConf.getInt(HBASE_BLOCKING_STORE_FILES, value);
- if (blockingStoreFiles != value) {
- blockingStoreFiles = value;
- tableDescriptor.setValue(BLOCKING_STORE_FILES_KEY, String.valueOf(value));
- LOG.info("Setting config property " + BLOCKING_STORE_FILES_KEY +
- " = " + blockingStoreFiles + " for " + tableName);
- return true;
- }
- return false;
- }
-
- protected String getSplitPointsStr(String splitPoints) {
- if (StringUtils.isEmpty(splitPoints.trim())) {
- return "";
- }
- String[] points = splitPoints.split(",");
- if (points.length > 0) {
- StringBuilder sb = new StringBuilder(" SPLIT ON ");
- sb.append("(");
- for (String point : points) {
- sb.append("'");
- sb.append(point.trim());
- sb.append("'");
- sb.append(",");
- }
- sb.deleteCharAt(sb.length() - 1);
- sb.append(")");
- return sb.toString();
- }
- return "";
- }
-
- /**
- * Insert precision YARN container data.
- */
- public void insertContainerMetrics(List<ContainerMetric> metrics)
- throws SQLException, IOException {
- Connection conn = getConnection();
- PreparedStatement metricRecordStmt = null;
-
- try {
- metricRecordStmt = conn.prepareStatement(
- String.format(UPSERT_CONTAINER_METRICS_SQL, CONTAINER_METRICS_TABLE_NAME));
- for (ContainerMetric metric : metrics) {
- metricRecordStmt.clearParameters();
- metricRecordStmt.setString(1, ContainerId.fromString(metric.getContainerId())
- .getApplicationAttemptId().getApplicationId().toString());
- metricRecordStmt.setString(2, metric.getContainerId());
- metricRecordStmt.setTimestamp(3, new Timestamp(metric.getStartTime()));
- metricRecordStmt.setTimestamp(4, new Timestamp(metric.getFinishTime()));
- metricRecordStmt.setLong(5, metric.getFinishTime() - metric.getStartTime());
- metricRecordStmt.setString(6, metric.getHostName());
- metricRecordStmt.setInt(7, metric.getExitCode());
- metricRecordStmt.setLong(8, metric.getLocalizationDuration());
- metricRecordStmt.setLong(9, metric.getLaunchDuration());
- metricRecordStmt.setDouble(10, (double) metric.getPmemLimit() / 1024);
- metricRecordStmt.setDouble(11,
- ((double) metric.getPmemLimit() / 1024) * (metric.getFinishTime()
- - metric.getStartTime()));
- metricRecordStmt.setDouble(12, (double) metric.getVmemLimit() / 1024);
- metricRecordStmt.setDouble(13, (double) metric.getPmemUsedMin() / 1024);
- metricRecordStmt.setDouble(14, (double) metric.getPmemUsedMax() / 1024);
- metricRecordStmt.setDouble(15, (double) metric.getPmemUsedAvg() / 1024);
- metricRecordStmt.setDouble(16, (double) metric.getPmem50Pct() / 1024);
- metricRecordStmt.setDouble(17, (double) metric.getPmem75Pct() / 1024);
- metricRecordStmt.setDouble(18, (double) metric.getPmem90Pct() / 1024);
- metricRecordStmt.setDouble(19, (double) metric.getPmem95Pct()/ 1024);
- metricRecordStmt.setDouble(20, (double) metric.getPmem99Pct() / 1024);
- metricRecordStmt.setDouble(21, (double) metric.getPmemLimit() / 1024
- - (double) metric.getPmemUsedMax() / 1024);
- metricRecordStmt.setDouble(22, ((double) metric.getPmemLimit() / 1024
- - (double) metric.getPmemUsedMax() / 1024) * (metric.getFinishTime()
- - metric.getStartTime()));
-
- try {
- metricRecordStmt.executeUpdate();
- } catch (SQLException sql) {
- LOG.error("Failed on insert records to store.", sql);
- }
- }
-
- conn.commit();
- } finally {
- if (metricRecordStmt != null) {
- try {
- metricRecordStmt.close();
- } catch (SQLException e) {
- // Ignore
- }
- }
- if (conn != null) {
- try {
- conn.close();
- } catch (SQLException sql) {
- // Ignore
- }
- }
- }
- }
-
- /**
- * Insert precision data.
- */
- public void insertMetricRecordsWithMetadata(TimelineMetricMetadataManager metadataManager,
- TimelineMetrics metrics, boolean skipCache) throws SQLException, IOException {
- List<TimelineMetric> timelineMetrics = metrics.getMetrics();
- if (timelineMetrics == null || timelineMetrics.isEmpty()) {
- LOG.debug("Empty metrics insert request.");
- return;
- }
- for (Iterator<TimelineMetric> iterator = timelineMetrics.iterator(); iterator.hasNext();) {
-
- TimelineMetric tm = iterator.next();
-
- boolean acceptMetric = TimelineMetricsFilter.acceptMetric(tm);
-
- // Write to metadata cache on successful write to store
- if (metadataManager != null) {
- metadataManager.putIfModifiedTimelineMetricMetadata(
- metadataManager.getTimelineMetricMetadata(tm, acceptMetric));
-
- metadataManager.putIfModifiedHostedAppsMetadata(
- tm.getHostName(), tm.getAppId());
-
- if (!tm.getAppId().equals("FLUME_HANDLER")) {
- metadataManager.putIfModifiedHostedInstanceMetadata(tm.getInstanceId(), tm.getHostName());
- }
- }
- if (!acceptMetric) {
- iterator.remove();
- }
- }
-
- if (!skipCache && cacheEnabled) {
- LOG.debug("Adding metrics to cache");
- if (insertCache.size() >= cacheSize) {
- commitMetricsFromCache();
- }
- try {
- insertCache.put(metrics); // blocked while the queue is full
- } catch (InterruptedException e) {
- e.printStackTrace();
- }
- } else {
- LOG.debug("Skipping metrics cache");
- commitMetrics(metrics);
- }
- }
-
- public void insertMetricRecords(TimelineMetrics metrics, boolean skipCache) throws SQLException, IOException {
- insertMetricRecordsWithMetadata(null, metrics, skipCache);
- }
-
- public void insertMetricRecords(TimelineMetrics metrics) throws SQLException, IOException {
- insertMetricRecords(metrics, false);
- }
-
-
- @SuppressWarnings("unchecked")
- public TimelineMetrics getMetricRecords(
- final Condition condition, Multimap<String, List<Function>> metricFunctions)
- throws SQLException, IOException {
-
- validateConditionIsNotEmpty(condition);
-
- Connection conn = getConnection();
- PreparedStatement stmt = null;
- ResultSet rs = null;
- TimelineMetrics metrics = new TimelineMetrics();
-
- try {
- //get latest
- if (condition.isPointInTime()){
- getLatestMetricRecords(condition, conn, metrics);
- } else {
- if (condition.getEndTime() >= condition.getStartTime()) {
- stmt = PhoenixTransactSQL.prepareGetMetricsSqlStmt(conn, condition);
- rs = stmt.executeQuery();
- while (rs.next()) {
- appendMetricFromResultSet(metrics, condition, metricFunctions, rs);
- }
- } else {
- LOG.warn("Skipping metrics query because endTime < startTime");
- }
- }
-
- } catch (PhoenixIOException pioe) {
- Throwable pioe2 = pioe.getCause();
- // Need to find out if this is exception "Could not find hash cache
- // for joinId" or another PhoenixIOException
- if (pioe2 instanceof PhoenixIOException &&
- pioe2.getCause() instanceof DoNotRetryIOException) {
- String className = null;
- for (StackTraceElement ste : pioe2.getCause().getStackTrace()) {
- className = ste.getClassName();
- }
-
- if (className != null && className.equals("HashJoinRegionScanner")) {
- LOG.error("The cache might have expired and have been removed. Try to" +
- " increase the cache size by setting bigger value for " +
- "phoenix.coprocessor.maxMetaDataCacheSize in ams-hbase-site config." +
- " Falling back to sort-merge join algorithm.");
- PhoenixTransactSQL.setSortMergeJoinEnabled(true);
- }
- }
- throw pioe;
- } catch (RuntimeException ex) {
- // We need to find out if this is a real IO exception
- // or exception "maxStamp is smaller than minStamp"
- // which is thrown in hbase TimeRange.java
- Throwable io = ex.getCause();
- String className = null;
- if (io != null) {
- for (StackTraceElement ste : io.getStackTrace()) {
- className = ste.getClassName();
- }
- }
- if (className != null && className.equals("TimeRange")) {
- // This is "maxStamp is smaller than minStamp" exception
- // Log error and return empty metrics
- LOG.debug(io);
- return new TimelineMetrics();
- } else {
- throw ex;
- }
-
- } finally {
- if (rs != null) {
- try {
- rs.close();
- } catch (SQLException e) {
- // Ignore
- }
- }
- if (stmt != null) {
- try {
- stmt.close();
- } catch (SQLException e) {
- // Ignore
- }
- }
- if (conn != null) {
- try {
- conn.close();
- } catch (SQLException sql) {
- // Ignore
- }
- }
- }
-
- LOG.debug("Metrics records size: " + metrics.getMetrics().size());
- return metrics;
- }
-
- /**
- * Apply aggregate function to the result if supplied else get precision
- * or aggregate data with default function applied.
- */
- private void appendMetricFromResultSet(TimelineMetrics metrics, Condition condition,
- Multimap<String, List<Function>> metricFunctions,
- ResultSet rs) throws SQLException, IOException {
- byte[] uuid = rs.getBytes("UUID");
- String metricName = metadataManagerInstance.getMetricNameFromUuid(uuid);
- Collection<List<Function>> functionList = findMetricFunctions(metricFunctions, metricName);
-
- for (List<Function> functions : functionList) {
- // Apply aggregation function if present
- if ((functions != null && !functions.isEmpty())) {
- if (functions.size() > 1) {
- throw new IllegalArgumentException("Multiple aggregate functions not supported.");
- }
- for (Function f : functions) {
- if (f.getReadFunction() == Function.ReadFunction.VALUE) {
- getTimelineMetricsFromResultSet(metrics, f, condition, rs);
- } else {
- SingleValuedTimelineMetric metric =
- TIMELINE_METRIC_READ_HELPER.getAggregatedTimelineMetricFromResultSet(rs, f);
-
- if (condition.isGrouped()) {
- metrics.addOrMergeTimelineMetric(metric);
- } else {
- metrics.getMetrics().add(metric.getTimelineMetric());
- }
- }
- }
- } else {
- // No aggregation requested
- // Execution never goes here, function always contain at least 1 element
- getTimelineMetricsFromResultSet(metrics, null, condition, rs);
- }
- }
- }
-
- private void getTimelineMetricsFromResultSet(TimelineMetrics metrics, Function f, Condition condition, ResultSet rs) throws SQLException, IOException {
- if (condition.getPrecision().equals(Precision.SECONDS)) {
- TimelineMetric metric = TIMELINE_METRIC_READ_HELPER.getTimelineMetricFromResultSet(rs);
- if (f != null && f.getSuffix() != null) { //Case : Requesting "._rate" for precision data
- metric.setMetricName(metric.getMetricName() + f.getSuffix());
- }
- if (condition.isGrouped()) {
- metrics.addOrMergeTimelineMetric(metric);
- } else {
- metrics.getMetrics().add(metric);
- }
-
- } else {
- SingleValuedTimelineMetric metric =
- TIMELINE_METRIC_READ_HELPER.getAggregatedTimelineMetricFromResultSet(rs, f);
- if (condition.isGrouped()) {
- metrics.addOrMergeTimelineMetric(metric);
- } else {
- metrics.getMetrics().add(metric.getTimelineMetric());
- }
- }
- }
-
- private void getLatestMetricRecords(Condition condition, Connection conn,
- TimelineMetrics metrics) throws SQLException, IOException {
-
- validateConditionIsNotEmpty(condition);
-
- PreparedStatement stmt = PhoenixTransactSQL.prepareGetLatestMetricSqlStmt(conn, condition);
- ResultSet rs = null;
- try {
- rs = stmt.executeQuery();
- while (rs.next()) {
- TimelineMetric metric = getLastTimelineMetricFromResultSet(rs);
- metrics.getMetrics().add(metric);
- }
- } finally {
- if (rs != null) {
- try {
- rs.close();
- } catch (SQLException e) {
- // Ignore
- }
- }
- if (stmt != null) {
- stmt.close();
- }
- }
- }
-
- /**
- * Get metrics aggregated across hosts.
- *
- * @param condition @Condition
- * @return @TimelineMetrics
- * @throws SQLException
- */
- public TimelineMetrics getAggregateMetricRecords(final Condition condition,
- Multimap<String, List<Function>> metricFunctions) throws SQLException {
-
- validateConditionIsNotEmpty(condition);
-
- Connection conn = getConnection();
- PreparedStatement stmt = null;
- ResultSet rs = null;
- TimelineMetrics metrics = new TimelineMetrics();
-
- try {
- //get latest
- if(condition.isPointInTime()) {
- getLatestAggregateMetricRecords(condition, conn, metrics, metricFunctions);
- } else {
- stmt = PhoenixTransactSQL.prepareGetAggregateSqlStmt(conn, condition);
-
- rs = stmt.executeQuery();
- while (rs.next()) {
- appendAggregateMetricFromResultSet(metrics, condition, metricFunctions, rs);
- }
- }
- } finally {
- if (rs != null) {
- try {
- rs.close();
- } catch (SQLException e) {
- // Ignore
- }
- }
- if (stmt != null) {
- try {
- stmt.close();
- } catch (SQLException e) {
- // Ignore
- }
- }
- if (conn != null) {
- try {
- conn.close();
- } catch (SQLException sql) {
- // Ignore
- }
- }
- }
-
- LOG.debug("Aggregate records size: " + metrics.getMetrics().size());
- return metrics;
- }
-
- private void appendAggregateMetricFromResultSet(TimelineMetrics metrics,
- Condition condition, Multimap<String, List<Function>> metricFunctions,
- ResultSet rs) throws SQLException {
-
- byte[] uuid = rs.getBytes("UUID");
- String metricName = metadataManagerInstance.getMetricNameFromUuid(uuid);
- Collection<List<Function>> functionList = findMetricFunctions(metricFunctions, metricName);
-
- for (List<Function> functions : functionList) {
- for (Function aggregateFunction : functions) {
- SingleValuedTimelineMetric metric;
-
- if (condition.getPrecision() == Precision.MINUTES
- || condition.getPrecision() == Precision.HOURS
- || condition.getPrecision() == Precision.DAYS) {
- metric = getAggregateTimelineMetricFromResultSet(rs, aggregateFunction, false);
- } else {
- metric = getAggregateTimelineMetricFromResultSet(rs, aggregateFunction, true);
- }
-
- if (condition.isGrouped()) {
- metrics.addOrMergeTimelineMetric(metric);
- } else {
- metrics.getMetrics().add(metric.getTimelineMetric());
- }
- }
- }
-
- }
-
- private void getLatestAggregateMetricRecords(Condition condition,
- Connection conn, TimelineMetrics metrics,
- Multimap<String, List<Function>> metricFunctions) throws SQLException {
-
- PreparedStatement stmt = null;
- SplitByMetricNamesCondition splitCondition =
- new SplitByMetricNamesCondition(condition);
-
- for (byte[] uuid: condition.getUuids()) {
-
- splitCondition.setCurrentUuid(uuid);
- stmt = PhoenixTransactSQL.prepareGetLatestAggregateMetricSqlStmt(conn, splitCondition);
- ResultSet rs = null;
- try {
- rs = stmt.executeQuery();
- while (rs.next()) {
- String metricName = metadataManagerInstance.getMetricNameFromUuid(uuid);
- Collection<List<Function>> functionList = findMetricFunctions(metricFunctions, metricName);
- for (List<Function> functions : functionList) {
- if (functions != null) {
- for (Function f : functions) {
- SingleValuedTimelineMetric metric =
- getAggregateTimelineMetricFromResultSet(rs, f, true);
-
- if (condition.isGrouped()) {
- metrics.addOrMergeTimelineMetric(metric);
- } else {
- metrics.getMetrics().add(metric.getTimelineMetric());
- }
- }
- } else {
- SingleValuedTimelineMetric metric =
- getAggregateTimelineMetricFromResultSet(rs, new Function(), true);
- metrics.getMetrics().add(metric.getTimelineMetric());
- }
- }
- }
- } finally {
- if (rs != null) {
- try {
- rs.close();
- } catch (SQLException e) {
- // Ignore
- }
- }
- if (stmt != null) {
- stmt.close();
- }
- }
- }
- }
-
- private SingleValuedTimelineMetric getAggregateTimelineMetricFromResultSet(ResultSet rs,
- Function f, boolean useHostCount) throws SQLException {
-
- String countColumnName = "METRIC_COUNT";
- if (useHostCount) {
- countColumnName = "HOSTS_COUNT";
- }
-
- byte[] uuid = rs.getBytes("UUID");
- TimelineMetric timelineMetric = metadataManagerInstance.getMetricFromUuid(uuid);
-
- SingleValuedTimelineMetric metric = new SingleValuedTimelineMetric(
- timelineMetric.getMetricName() + f.getSuffix(),
- timelineMetric.getAppId(),
- timelineMetric.getInstanceId(),
- null,
- rs.getLong("SERVER_TIME")
- );
-
- double value;
- switch(f.getReadFunction()){
- case AVG:
- value = rs.getDouble("METRIC_SUM") / rs.getInt(countColumnName);
- break;
- case MIN:
- value = rs.getDouble("METRIC_MIN");
- break;
- case MAX:
- value = rs.getDouble("METRIC_MAX");
- break;
- case SUM:
- value = rs.getDouble("METRIC_SUM");
- break;
- default:
- value = rs.getDouble("METRIC_SUM") / rs.getInt(countColumnName);
- break;
- }
-
- metric.setSingleTimeseriesValue(rs.getLong("SERVER_TIME"), value);
-
- return metric;
- }
-
- private void validateConditionIsNotEmpty(Condition condition) {
- if (condition.isEmpty()) {
- throw new IllegalArgumentException("No filter criteria specified.");
- }
- }
-
- private Collection<List<Function>> findMetricFunctions(Multimap<String, List<Function>> metricFunctions,
- String metricName) {
- if (metricFunctions.containsKey(metricName)) {
- return metricFunctions.get(metricName);
- }
-
- for (String metricNameEntry : metricFunctions.keySet()) {
-
- String metricRegEx;
- //Special case handling for metric name with * and __%.
- //For example, dfs.NNTopUserOpCounts.windowMs=300000.op=*.user=%.count
- // or dfs.NNTopUserOpCounts.windowMs=300000.op=__%.user=%.count
- if (metricNameEntry.contains("*") || metricNameEntry.contains("__%")) {
- String metricNameWithEscSeq = metricNameEntry.replace("*", "\\*").replace("__%", "..%");
- metricRegEx = metricNameWithEscSeq.replace("%", ".*");
- } else {
- metricRegEx = metricNameEntry.replace("%", ".*");
- }
- if (metricName.matches(metricRegEx)) {
- return metricFunctions.get(metricNameEntry);
- }
- }
-
- return null;
- }
-
- public void saveHostAggregateRecords(Map<TimelineMetric, MetricHostAggregate> hostAggregateMap,
- String phoenixTableName) throws SQLException {
-
- if (hostAggregateMap == null || hostAggregateMap.isEmpty()) {
- LOG.debug("Empty aggregate records.");
- return;
- }
-
- Connection conn = getConnection();
- PreparedStatement stmt = null;
-
- long start = System.currentTimeMillis();
- int rowCount = 0;
-
- try {
- stmt = conn.prepareStatement(
- String.format(UPSERT_AGGREGATE_RECORD_SQL, phoenixTableName));
-
- for (Map.Entry<TimelineMetric, MetricHostAggregate> metricAggregate :
- hostAggregateMap.entrySet()) {
-
- TimelineMetric metric = metricAggregate.getKey();
- MetricHostAggregate hostAggregate = metricAggregate.getValue();
-
- byte[] uuid = metadataManagerInstance.getUuid(metric);
- if (uuid == null) {
- LOG.error("Error computing UUID for metric. Cannot write metric : " + metric.toString());
- continue;
- }
- rowCount++;
- stmt.clearParameters();
- stmt.setBytes(1, uuid);
- stmt.setLong(2, metric.getStartTime());
- stmt.setDouble(3, hostAggregate.getSum());
- stmt.setDouble(4, hostAggregate.getMax());
- stmt.setDouble(5, hostAggregate.getMin());
- stmt.setDouble(6, hostAggregate.getNumberOfSamples());
-
- try {
- stmt.executeUpdate();
- } catch (SQLException sql) {
- LOG.error(sql);
- }
-
- if (rowCount >= PHOENIX_MAX_MUTATION_STATE_SIZE - 1) {
- conn.commit();
- rowCount = 0;
- }
-
- }
-
- conn.commit();
-
- } finally {
- if (stmt != null) {
- try {
- stmt.close();
- } catch (SQLException e) {
- // Ignore
- }
- }
- if (conn != null) {
- try {
- conn.close();
- } catch (SQLException sql) {
- // Ignore
- }
- }
- }
-
- long end = System.currentTimeMillis();
-
- if ((end - start) > 60000l) {
- LOG.info("Time to save map: " + (end - start) + ", " +
- "thread = " + Thread.currentThread().getClass());
- }
- if (aggregatorSink != null) {
- try {
- aggregatorSink.saveHostAggregateRecords(hostAggregateMap,
- getTablePrecision(phoenixTableName));
- } catch (Exception e) {
- LOG.warn(
- "Error writing host aggregate records metrics to external sink. "
- + e);
- }
- }
- }
-
- /**
- * Save Metric aggregate records.
- *
- * @throws SQLException
- */
- public void saveClusterAggregateRecords(Map<TimelineClusterMetric, MetricClusterAggregate> records)
- throws SQLException {
-
- if (records == null || records.isEmpty()) {
- LOG.debug("Empty aggregate records.");
- return;
- }
-
- long start = System.currentTimeMillis();
- String sqlStr = String.format(UPSERT_CLUSTER_AGGREGATE_SQL, METRICS_CLUSTER_AGGREGATE_TABLE_NAME);
- Connection conn = getConnection();
- PreparedStatement stmt = null;
- try {
- stmt = conn.prepareStatement(sqlStr);
- int rowCount = 0;
-
- for (Map.Entry<TimelineClusterMetric, MetricClusterAggregate>
- aggregateEntry : records.entrySet()) {
- TimelineClusterMetric clusterMetric = aggregateEntry.getKey();
- MetricClusterAggregate aggregate = aggregateEntry.getValue();
-
- if (LOG.isTraceEnabled()) {
- LOG.trace("clusterMetric = " + clusterMetric + ", " +
- "aggregate = " + aggregate);
- }
-
- rowCount++;
- byte[] uuid = metadataManagerInstance.getUuid(clusterMetric);
- if (uuid == null) {
- LOG.error("Error computing UUID for metric. Cannot write metrics : " + clusterMetric.toString());
- continue;
- }
- stmt.clearParameters();
- stmt.setBytes(1, uuid);
- stmt.setLong(2, clusterMetric.getTimestamp());
- stmt.setDouble(3, aggregate.getSum());
- stmt.setInt(4, aggregate.getNumberOfHosts());
- stmt.setDouble(5, aggregate.getMax());
- stmt.setDouble(6, aggregate.getMin());
-
- try {
- stmt.executeUpdate();
- } catch (SQLException sql) {
- // we have no way to verify it works!!!
- LOG.error(sql);
- }
-
- if (rowCount >= PHOENIX_MAX_MUTATION_STATE_SIZE - 1) {
- conn.commit();
- rowCount = 0;
- }
- }
-
- conn.commit();
-
- } finally {
- if (stmt != null) {
- try {
- stmt.close();
- } catch (SQLException e) {
- // Ignore
- }
- }
- if (conn != null) {
- try {
- conn.close();
- } catch (SQLException sql) {
- // Ignore
- }
- }
- }
- long end = System.currentTimeMillis();
- if ((end - start) > 60000l) {
- LOG.info("Time to save: " + (end - start) + ", " +
- "thread = " + Thread.currentThread().getName());
- }
- if (aggregatorSink != null) {
- try {
- aggregatorSink.saveClusterAggregateRecords(records);
- } catch (Exception e) {
- LOG.warn("Error writing cluster aggregate records metrics to external sink. ", e);
- }
- }
- }
-
-
- /**
- * Save Metric aggregate records.
- *
- * @throws SQLException
- */
- public void saveClusterAggregateRecordsSecond(Map<TimelineClusterMetric, MetricHostAggregate> records,
- String tableName) throws SQLException {
- if (records == null || records.isEmpty()) {
- LOG.debug("Empty aggregate records.");
- return;
- }
-
- long start = System.currentTimeMillis();
-
- Connection conn = getConnection();
- PreparedStatement stmt = null;
- try {
- stmt = conn.prepareStatement(String.format(UPSERT_CLUSTER_AGGREGATE_TIME_SQL, tableName));
- int rowCount = 0;
-
- for (Map.Entry<TimelineClusterMetric, MetricHostAggregate> aggregateEntry : records.entrySet()) {
- TimelineClusterMetric clusterMetric = aggregateEntry.getKey();
- MetricHostAggregate aggregate = aggregateEntry.getValue();
-
- if (LOG.isTraceEnabled()) {
- LOG.trace("clusterMetric = " + clusterMetric + ", " +
- "aggregate = " + aggregate);
- }
-
- byte[] uuid = metadataManagerInstance.getUuid(clusterMetric);
- if (uuid == null) {
- LOG.error("Error computing UUID for metric. Cannot write metric : " + clusterMetric.toString());
- continue;
- }
-
- rowCount++;
- stmt.clearParameters();
- stmt.setBytes(1, uuid);
- stmt.setLong(2, clusterMetric.getTimestamp());
- stmt.setDouble(3, aggregate.getSum());
- stmt.setLong(4, aggregate.getNumberOfSamples());
- stmt.setDouble(5, aggregate.getMax());
- stmt.setDouble(6, aggregate.getMin());
-
- try {
- stmt.executeUpdate();
- } catch (SQLException sql) {
- // we have no way to verify it works!!!
- LOG.error(sql);
- }
-
- if (rowCount >= PHOENIX_MAX_MUTATION_STATE_SIZE - 1) {
- conn.commit();
- rowCount = 0;
- }
- }
-
- conn.commit();
-
- } finally {
- if (stmt != null) {
- try {
- stmt.close();
- } catch (SQLException e) {
- // Ignore
- }
- }
- if (conn != null) {
- try {
- conn.close();
- } catch (SQLException sql) {
- // Ignore
- }
- }
- }
- long end = System.currentTimeMillis();
- if ((end - start) > 60000l) {
- LOG.info("Time to save: " + (end - start) + ", " +
- "thread = " + Thread.currentThread().getName());
- }
- if (aggregatorSink != null) {
- try {
- aggregatorSink.saveClusterTimeAggregateRecords(records,
- getTablePrecision(tableName));
- } catch (Exception e) {
- LOG.warn(
- "Error writing cluster time aggregate records metrics to external sink. "
- + e);
- }
- }
- }
-
- /**
- * Get precision for a table
- * @param tableName
- * @return precision
- */
- private Precision getTablePrecision(String tableName) {
- Precision tablePrecision = null;
- switch (tableName) {
- case METRICS_RECORD_TABLE_NAME:
- tablePrecision = Precision.SECONDS;
- break;
- case METRICS_AGGREGATE_MINUTE_TABLE_NAME:
- case METRICS_CLUSTER_AGGREGATE_MINUTE_TABLE_NAME:
- tablePrecision = Precision.MINUTES;
- break;
- case METRICS_AGGREGATE_HOURLY_TABLE_NAME:
- case METRICS_CLUSTER_AGGREGATE_HOURLY_TABLE_NAME:
- tablePrecision = Precision.HOURS;
- break;
- case METRICS_AGGREGATE_DAILY_TABLE_NAME:
- case METRICS_CLUSTER_AGGREGATE_DAILY_TABLE_NAME:
- tablePrecision = Precision.DAYS;
- break;
- }
- return tablePrecision;
- }
-
- /**
- * Provide skip block cache hint for aggregator queries.
- */
- public boolean isSkipBlockCacheForAggregatorsEnabled() {
- return skipBlockCacheForAggregatorsEnabled;
- }
-
- /**
- * One time save of metadata when discovering topology during aggregation.
- * @throws SQLException
- */
- public void saveHostAppsMetadata(Map<String, TimelineMetricHostMetadata> hostMetadata) throws SQLException {
- Connection conn = getConnection();
- PreparedStatement stmt = null;
- try {
- stmt = conn.prepareStatement(UPSERT_HOSTED_APPS_METADATA_SQL);
- int rowCount = 0;
-
- for (Map.Entry<String, TimelineMetricHostMetadata> hostedAppsEntry : hostMetadata.entrySet()) {
- TimelineMetricHostMetadata timelineMetricHostMetadata = hostedAppsEntry.getValue();
- if (LOG.isTraceEnabled()) {
- LOG.trace("HostedAppsMetadata: " + hostedAppsEntry);
- }
-
- stmt.clearParameters();
- stmt.setString(1, hostedAppsEntry.getKey());
- stmt.setBytes(2, timelineMetricHostMetadata.getUuid());
- stmt.setString(3, StringUtils.join(timelineMetricHostMetadata.getHostedApps().keySet(), ","));
- try {
- stmt.executeUpdate();
- rowCount++;
- } catch (SQLException sql) {
- LOG.error("Error saving hosted apps metadata.", sql);
- }
- }
-
- conn.commit();
- LOG.info("Saved " + rowCount + " hosted apps metadata records.");
-
- } finally {
- if (stmt != null) {
- try {
- stmt.close();
- } catch (SQLException e) {
- // Ignore
- }
- }
- if (conn != null) {
- try {
- conn.close();
- } catch (SQLException sql) {
- // Ignore
- }
- }
- }
- }
-
- public void saveInstanceHostsMetadata(Map<String, Set<String>> instanceHostsMap) throws SQLException {
- Connection conn = getConnection();
- PreparedStatement stmt = null;
- try {
- stmt = conn.prepareStatement(UPSERT_INSTANCE_HOST_METADATA_SQL);
- int rowCount = 0;
-
- for (Map.Entry<String, Set<String>> hostInstancesEntry : instanceHostsMap.entrySet()) {
- if (LOG.isTraceEnabled()) {
- LOG.trace("Host Instances Entry: " + hostInstancesEntry);
- }
-
- String instanceId = hostInstancesEntry.getKey();
-
- for(String hostname : hostInstancesEntry.getValue()) {
- stmt.clearParameters();
- stmt.setString(1, instanceId);
- stmt.setString(2, hostname);
- try {
- stmt.executeUpdate();
- rowCount++;
- } catch (SQLException sql) {
- LOG.error("Error saving host instances metadata.", sql);
- }
- }
-
- }
-
- conn.commit();
- LOG.info("Saved " + rowCount + " host instances metadata records.");
-
- } finally {
- if (stmt != null) {
- try {
- stmt.close();
- } catch (SQLException e) {
- // Ignore
- }
- }
- if (conn != null) {
- try {
- conn.close();
- } catch (SQLException sql) {
- // Ignore
- }
- }
- }
- }
-
- /**
- * Save metdata on updates.
- * @param metricMetadata @Collection<@TimelineMetricMetadata>
- * @throws SQLException
- */
- public void saveMetricMetadata(Collection<TimelineMetricMetadata> metricMetadata) throws SQLException {
- if (metricMetadata.isEmpty()) {
- LOG.info("No metadata records to save.");
- return;
- }
-
- Connection conn = getConnection();
- PreparedStatement stmt = null;
-
- try {
- stmt = conn.prepareStatement(UPSERT_METADATA_SQL);
- int rowCount = 0;
-
- for (TimelineMetricMetadata metadata : metricMetadata) {
- if (LOG.isTraceEnabled()) {
- LOG.trace("TimelineMetricMetadata: metricName = " + metadata.getMetricName()
- + ", appId = " + metadata.getAppId()
- + ", seriesStartTime = " + metadata.getSeriesStartTime()
- );
- }
- try {
- stmt.clearParameters();
- stmt.setString(1, metadata.getMetricName());
- stmt.setString(2, metadata.getAppId());
- stmt.setString(3, metadata.getInstanceId());
- stmt.setBytes(4, metadata.getUuid());
- stmt.setString(5, metadata.getUnits());
- stmt.setString(6, metadata.getType());
- stmt.setLong(7, metadata.getSeriesStartTime());
- stmt.setBoolean(8, metadata.isSupportsAggregates());
- stmt.setBoolean(9, metadata.isWhitelisted());
- } catch (Exception e) {
- LOG.error("Exception in saving metric metadata entry. ");
- continue;
- }
-
- try {
- stmt.executeUpdate();
- rowCount++;
- } catch (SQLException sql) {
- LOG.error("Error saving metadata.", sql);
- }
- }
-
- conn.commit();
- LOG.info("Saved " + rowCount + " metadata records.");
-
- } finally {
- if (stmt != null) {
- try {
- stmt.close();
- } catch (SQLException e) {
- // Ignore
- }
- }
- if (conn != null) {
- try {
- conn.close();
- } catch (SQLException sql) {
- // Ignore
- }
- }
- }
- }
-
- public Map<String, TimelineMetricHostMetadata> getHostedAppsMetadata() throws SQLException {
- Map<String, TimelineMetricHostMetadata> hostedAppMap = new HashMap<>();
- Connection conn = getConnection();
- PreparedStatement stmt = null;
- ResultSet rs = null;
-
- try {
- stmt = conn.prepareStatement(GET_HOSTED_APPS_METADATA_SQL);
- rs = stmt.executeQuery();
-
- while (rs.next()) {
- TimelineMetricHostMetadata hostMetadata = new TimelineMetricHostMetadata(new HashSet<>(Arrays.asList(StringUtils.split(rs.getString("APP_IDS"), ","))));
- hostMetadata.setUuid(rs.getBytes("UUID"));
- hostedAppMap.put(rs.getString("HOSTNAME"), hostMetadata);
- }
-
- } finally {
- if (rs != null) {
- try {
- rs.close();
- } catch (SQLException e) {
- // Ignore
- }
- }
- if (stmt != null) {
- try {
- stmt.close();
- } catch (SQLException e) {
- // Ignore
- }
- }
- if (conn != null) {
- try {
- conn.close();
- } catch (SQLException sql) {
- // Ignore
- }
- }
- }
-
- return hostedAppMap;
- }
-
- public Map<String, Set<String>> getInstanceHostsMetdata() throws SQLException {
- Map<String, Set<String>> instanceHostsMap = new HashMap<>();
- Connection conn = getConnection();
- PreparedStatement stmt = null;
- ResultSet rs = null;
-
- try {
- stmt = conn.prepareStatement(GET_INSTANCE_HOST_METADATA_SQL);
- rs = stmt.executeQuery();
-
- while (rs.next()) {
- String instanceId = rs.getString("INSTANCE_ID");
- String hostname = rs.getString("HOSTNAME");
-
- if (!instanceHostsMap.containsKey(instanceId)) {
- instanceHostsMap.put(instanceId, new HashSet<String>());
- }
- instanceHostsMap.get(instanceId).add(hostname);
- }
-
- } finally {
- if (rs != null) {
- try {
- rs.close();
- } catch (SQLException e) {
- // Ignore
- }
- }
- if (stmt != null) {
- try {
- stmt.close();
- } catch (SQLException e) {
- // Ignore
- }
- }
- if (conn != null) {
- try {
- conn.close();
- } catch (SQLException sql) {
- // Ignore
- }
- }
- }
-
- return instanceHostsMap;
- }
-
- // No filter criteria support for now.
- public Map<TimelineMetricMetadataKey, TimelineMetricMetadata> getTimelineMetricMetadata() throws SQLException {
- Map<TimelineMetricMetadataKey, TimelineMetricMetadata> metadataMap = new HashMap<>();
- Connection conn = getConnection();
- PreparedStatement stmt = null;
- ResultSet rs = null;
-
- try {
- stmt = conn.prepareStatement(GET_METRIC_METADATA_SQL);
- rs = stmt.executeQuery();
-
- while (rs.next()) {
- String metricName = rs.getString("METRIC_NAME");
- String appId = rs.getString("APP_ID");
- String instanceId = rs.getString("INSTANCE_ID");
- TimelineMetricMetadata metadata = new TimelineMetricMetadata(
- metricName,
- appId,
- instanceId,
- rs.getString("UNITS"),
- rs.getString("TYPE"),
- rs.getLong("START_TIME"),
- rs.getBoolean("SUPPORTS_AGGREGATION"),
- rs.getBoolean("IS_WHITELISTED")
- );
-
- TimelineMetricMetadataKey key = new TimelineMetricMetadataKey(metricName, appId, instanceId);
- metadata.setIsPersisted(true); // Always true on retrieval
- metadata.setUuid(rs.getBytes("UUID"));
- metadataMap.put(key, metadata);
- }
-
- } finally {
- if (rs != null) {
- try {
- rs.close();
- } catch (SQLException e) {
- // Ignore
- }
- }
- if (stmt != null) {
- try {
- stmt.close();
- } catch (SQLException e) {
- // Ignore
- }
- }
- if (conn != null) {
- try {
- conn.close();
- } catch (SQLException sql) {
- // Ignore
- }
- }
- }
-
- return metadataMap;
- }
-
- public void setMetadataInstance(TimelineMetricMetadataManager metadataManager) {
- this.metadataManagerInstance = metadataManager;
- TIMELINE_METRIC_READ_HELPER = new TimelineMetricReadHelper(this.metadataManagerInstance);
- }
-}
[24/28] ambari git commit: AMBARI-22740 : Fix integration test for
HBase in branch-3.0-ams due to UUID changes. (avijayan)
Posted by av...@apache.org.
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/aggregators/AggregatorUtils.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/aggregators/AggregatorUtils.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/aggregators/AggregatorUtils.java
new file mode 100644
index 0000000..43c1f2b
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/aggregators/AggregatorUtils.java
@@ -0,0 +1,254 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.metrics.core.timeline.aggregators;
+
+
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.TreeMap;
+
+import org.apache.commons.collections.MapUtils;
+import org.apache.commons.lang.StringUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.metrics2.sink.timeline.PostProcessingUtil;
+import org.apache.hadoop.metrics2.sink.timeline.TimelineMetric;
+
+/**
+ *
+ */
+public class AggregatorUtils {
+
+ private static final Log LOG = LogFactory.getLog(AggregatorUtils.class);
+
+ public static double[] calculateAggregates(Map<Long, Double> metricValues) {
+ double[] values = new double[4];
+ double max = Double.MIN_VALUE;
+ double min = Double.MAX_VALUE;
+ double sum = 0.0;
+ int metricCount = 0;
+
+ if (metricValues != null && !metricValues.isEmpty()) {
+ for (Double value : metricValues.values()) {
+ // TODO: Some nulls in data - need to investigate null values from host
+ if (value != null) {
+ if (value > max) {
+ max = value;
+ }
+ if (value < min) {
+ min = value;
+ }
+ sum += value;
+ }
+ }
+ metricCount = metricValues.values().size();
+ }
+ // BR: WHY ZERO is a good idea?
+ values[0] = sum;
+ values[1] = max != Double.MIN_VALUE ? max : 0.0;
+ values[2] = min != Double.MAX_VALUE ? min : 0.0;
+ values[3] = metricCount;
+
+ return values;
+ }
+
+ public static Map<TimelineClusterMetric, Double> sliceFromTimelineMetric(
+ TimelineMetric timelineMetric, List<Long[]> timeSlices, boolean interpolationEnabled) {
+
+ if (timelineMetric.getMetricValues().isEmpty()) {
+ return null;
+ }
+
+ Map<TimelineClusterMetric, Double> timelineClusterMetricMap =
+ new HashMap<>();
+
+ Long prevTimestamp = -1l;
+ TimelineClusterMetric prevMetric = null;
+ int count = 0;
+ double sum = 0.0;
+
+ Map<Long,Double> timeSliceValueMap = new HashMap<>();
+ for (Map.Entry<Long, Double> metric : timelineMetric.getMetricValues().entrySet()) {
+ if (metric.getValue() == null) {
+ continue;
+ }
+
+ Long timestamp = getSliceTimeForMetric(timeSlices, Long.parseLong(metric.getKey().toString()));
+ if (timestamp != -1) {
+ // Metric is within desired time range
+ TimelineClusterMetric clusterMetric = new TimelineClusterMetric(
+ timelineMetric.getMetricName(),
+ timelineMetric.getAppId(),
+ timelineMetric.getInstanceId(),
+ timestamp);
+
+ if (prevTimestamp < 0 || timestamp.equals(prevTimestamp)) {
+ Double newValue = metric.getValue();
+ if (newValue > 0.0) {
+ sum += newValue;
+ count++;
+ }
+ } else {
+ double metricValue = (count > 0) ? (sum / count) : 0.0;
+ timelineClusterMetricMap.put(prevMetric, metricValue);
+ timeSliceValueMap.put(prevMetric.getTimestamp(), metricValue);
+ sum = metric.getValue();
+ count = sum > 0.0 ? 1 : 0;
+ }
+
+ prevTimestamp = timestamp;
+ prevMetric = clusterMetric;
+ }
+ }
+
+ if (prevTimestamp > 0) {
+ double metricValue = (count > 0) ? (sum / count) : 0.0;
+ timelineClusterMetricMap.put(prevMetric, metricValue);
+ timeSliceValueMap.put(prevTimestamp, metricValue);
+ }
+
+ if (interpolationEnabled) {
+ Map<Long, Double> interpolatedValues = interpolateMissingPeriods(timelineMetric.getMetricValues(), timeSlices, timeSliceValueMap, timelineMetric.getType());
+ for (Map.Entry<Long, Double> entry : interpolatedValues.entrySet()) {
+ TimelineClusterMetric timelineClusterMetric = new TimelineClusterMetric(timelineMetric.getMetricName(), timelineMetric.getAppId(), timelineMetric.getInstanceId(), entry.getKey());
+ timelineClusterMetricMap.putIfAbsent(timelineClusterMetric, entry.getValue());
+ }
+ }
+
+ return timelineClusterMetricMap;
+ }
+
+ private static Map<Long, Double> interpolateMissingPeriods(TreeMap<Long, Double> metricValues,
+ List<Long[]> timeSlices,
+ Map<Long, Double> timeSliceValueMap, String type) {
+ Map<Long, Double> resultClusterMetricMap = new HashMap<>();
+
+ if (StringUtils.isNotEmpty(type) && "COUNTER".equalsIgnoreCase(type)) {
+ //For Counter Based metrics, ok to do interpolation and extrapolation
+
+ List<Long> requiredTimestamps = new ArrayList<>();
+ for (Long[] timeSlice : timeSlices) {
+ if (!timeSliceValueMap.containsKey(timeSlice[1])) {
+ requiredTimestamps.add(timeSlice[1]);
+ }
+ }
+ Map<Long, Double> interpolatedValuesMap = PostProcessingUtil.interpolate(metricValues, requiredTimestamps);
+
+ if (interpolatedValuesMap != null) {
+ for (Map.Entry<Long, Double> entry : interpolatedValuesMap.entrySet()) {
+ Double interpolatedValue = entry.getValue();
+
+ if (interpolatedValue != null) {
+ resultClusterMetricMap.put( entry.getKey(), interpolatedValue);
+ } else {
+ LOG.debug("Cannot compute interpolated value, hence skipping.");
+ }
+ }
+ }
+ } else {
+ //For other metrics, ok to do only interpolation
+
+ Double defaultNextSeenValue = null;
+ if (MapUtils.isEmpty(timeSliceValueMap) && MapUtils.isNotEmpty(metricValues)) {
+ //If no value was found within the start_time based slices, but the metric has value in the server_time range,
+ // use that.
+
+ Map.Entry<Long,Double> firstEntry = metricValues.firstEntry();
+ defaultNextSeenValue = firstEntry.getValue();
+ LOG.debug("Found a data point outside timeslice range: " + new Date(firstEntry.getKey()) + ": " + defaultNextSeenValue);
+ }
+
+ for (int sliceNum = 0; sliceNum < timeSlices.size(); sliceNum++) {
+ Long[] timeSlice = timeSlices.get(sliceNum);
+
+ if (!timeSliceValueMap.containsKey(timeSlice[1])) {
+ LOG.debug("Found an empty slice : " + new Date(timeSlice[0]) + ", " + new Date(timeSlice[1]));
+
+ Double lastSeenValue = null;
+ int index = sliceNum - 1;
+ Long[] prevTimeSlice = null;
+ while (lastSeenValue == null && index >= 0) {
+ prevTimeSlice = timeSlices.get(index--);
+ lastSeenValue = timeSliceValueMap.get(prevTimeSlice[1]);
+ }
+
+ Double nextSeenValue = null;
+ index = sliceNum + 1;
+ Long[] nextTimeSlice = null;
+ while (nextSeenValue == null && index < timeSlices.size()) {
+ nextTimeSlice = timeSlices.get(index++);
+ nextSeenValue = timeSliceValueMap.get(nextTimeSlice[1]);
+ }
+
+ if (nextSeenValue == null) {
+ nextSeenValue = defaultNextSeenValue;
+ }
+
+ Double interpolatedValue = PostProcessingUtil.interpolate(timeSlice[1],
+ (prevTimeSlice != null ? prevTimeSlice[1] : null), lastSeenValue,
+ (nextTimeSlice != null ? nextTimeSlice[1] : null), nextSeenValue);
+
+ if (interpolatedValue != null) {
+ LOG.debug("Interpolated value : " + interpolatedValue);
+ resultClusterMetricMap.put(timeSlice[1], interpolatedValue);
+ } else {
+ LOG.debug("Cannot compute interpolated value, hence skipping.");
+ }
+ }
+ }
+ }
+ return resultClusterMetricMap;
+ }
+
+ /**
+ * Return end of the time slice into which the metric fits.
+ */
+ public static Long getSliceTimeForMetric(List<Long[]> timeSlices, Long timestamp) {
+ for (Long[] timeSlice : timeSlices) {
+ if (timestamp >= timeSlice[0] && timestamp < timeSlice[1]) {
+ return timeSlice[1];
+ }
+ }
+ return -1l;
+ }
+
+ /**
+ * Return time slices to normalize the timeseries data.
+ */
+ public static List<Long[]> getTimeSlices(long startTime, long endTime, long timeSliceIntervalMillis) {
+ List<Long[]> timeSlices = new ArrayList<Long[]>();
+ long sliceStartTime = startTime;
+ while (sliceStartTime < endTime) {
+ timeSlices.add(new Long[] { sliceStartTime, sliceStartTime + timeSliceIntervalMillis });
+ sliceStartTime += timeSliceIntervalMillis;
+ }
+ return timeSlices;
+ }
+
+ public static long getRoundedCheckPointTimeMillis(long referenceTime, long aggregatorPeriod) {
+ return referenceTime - (referenceTime % aggregatorPeriod);
+ }
+
+ public static long getRoundedAggregateTimeMillis(long aggregatorPeriod) {
+ long currentTime = System.currentTimeMillis();
+ return currentTime - (currentTime % aggregatorPeriod);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/aggregators/CustomDownSampler.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/aggregators/CustomDownSampler.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/aggregators/CustomDownSampler.java
new file mode 100644
index 0000000..49e2bf6
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/aggregators/CustomDownSampler.java
@@ -0,0 +1,44 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.metrics.core.timeline.aggregators;
+
+import java.util.List;
+
+/**
+ * Interface to add a custom downsampler.
+ * Each configured downsampler will be executed during an aggregation cycle.
+ */
+public interface CustomDownSampler {
+
+ /**
+ * Gatekeeper to check the configs. If this fails, the downsampling is not done.
+ * @return
+ */
+ public boolean validateConfigs();
+
+ /**
+ * Return the set of statements that needs to be executed for the downsampling.
+ * @param startTime
+ * @param endTime
+ * @param tableName
+ * @return
+ */
+ public List<String> prepareDownSamplingStatement(Long startTime, Long endTime, String tableName);
+
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/aggregators/DownSamplerUtils.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/aggregators/DownSamplerUtils.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/aggregators/DownSamplerUtils.java
new file mode 100644
index 0000000..ad47931
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/aggregators/DownSamplerUtils.java
@@ -0,0 +1,117 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.metrics.core.timeline.aggregators;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+
+/**
+ * DownSampler utility class. Responsible for fetching downsampler configs from Metrics config, and determine if
+ * any downsamplers are configured.
+ */
+
+public class DownSamplerUtils {
+
+ public static final String downSamplerConfigPrefix = "timeline.metrics.downsampler.";
+ public static final String downSamplerMetricPatternsConfig = "metric.patterns";
+ public static final String topNDownSampler = "topn";
+ private static final Log LOG = LogFactory.getLog(DownSamplerUtils.class);
+
+
+
+ /**
+ * Get the list of metrics that are requested to be downsampled.
+ * @param configuration
+ * @return List of metric patterns/names that are to be downsampled.
+ */
+ public static List<String> getDownsampleMetricPatterns(Configuration configuration) {
+ Map<String, String> conf = configuration.getValByRegex(downSamplerConfigPrefix + "*");
+ List<String> metricPatterns = new ArrayList<>();
+ Set<String> keys = conf.keySet();
+ for (String key : keys) {
+ if (key.endsWith(downSamplerMetricPatternsConfig)) {
+ String patternString = conf.get(key);
+ String[] patterns = StringUtils.split(patternString, ",");
+ for (String pattern : patterns) {
+ if (StringUtils.isNotEmpty(pattern)) {
+ String trimmedPattern = pattern.trim();
+ metricPatterns.add(trimmedPattern);
+ }
+ }
+ }
+ }
+ return metricPatterns;
+ }
+
+ /**
+ * Get the list of downsamplers that are configured in ams-site
+ * Sample config
+ <name>timeline.metrics.downsampler.topn.metric.patterns</name>
+ <value>dfs.NNTopUserOpCounts.windowMs=60000.op%,dfs.NNTopUserOpCounts.windowMs=300000.op%</value>
+
+ <name>timeline.metrics.downsampler.topn.value</name>
+ <value>10</value>
+
+ <name>timeline.metrics.downsampler.topn.function</name>
+ <value>max</value>
+ * @param configuration
+ * @return
+ */
+ public static List<CustomDownSampler> getDownSamplers(Configuration configuration) {
+
+ Map<String,String> conf = configuration.getValByRegex(downSamplerConfigPrefix + "*");
+ List<CustomDownSampler> downSamplers = new ArrayList<>();
+ Set<String> keys = conf.keySet();
+
+ try {
+ for (String key : keys) {
+ if (key.startsWith(downSamplerConfigPrefix) && key.endsWith(downSamplerMetricPatternsConfig)) {
+ String type = key.split("\\.")[3];
+ CustomDownSampler downSampler = getDownSamplerByType(type, conf);
+ if (downSampler != null) {
+ downSamplers.add(downSampler);
+ }
+ }
+ }
+ } catch (Exception e) {
+ LOG.warn("Exception caught while parsing downsampler configs from ams-site : " + e.getMessage());
+ }
+ return downSamplers;
+ }
+
+ public static CustomDownSampler getDownSamplerByType(String type, Map<String, String> conf) {
+ if (type == null) {
+ return null;
+ }
+
+ if (StringUtils.isNotEmpty(type) && type.equalsIgnoreCase(topNDownSampler)) {
+ return TopNDownSampler.fromConfig(conf);
+ }
+
+ LOG.warn("Unknown downsampler requested : " + type);
+ return null;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/aggregators/Function.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/aggregators/Function.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/aggregators/Function.java
new file mode 100644
index 0000000..dd67b64
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/aggregators/Function.java
@@ -0,0 +1,214 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.metrics.core.timeline.aggregators;
+
+import org.apache.ambari.metrics.webapp.TimelineWebServices;
+
+import java.util.Arrays;
+
+/**
+ * Is used to determine metrics aggregate table.
+ *
+ * @see TimelineWebServices#getTimelineMetric
+ * @see TimelineWebServices#getTimelineMetrics
+ */
+public class Function {
+ public static Function DEFAULT_VALUE_FUNCTION = new Function(ReadFunction.VALUE, null);
+ private static final String SUFFIX_SEPARATOR = "\\._";
+
+ private ReadFunction readFunction = ReadFunction.VALUE;
+ private PostProcessingFunction postProcessingFunction = null;
+
+ public Function() {
+ }
+
+ public Function(ReadFunction readFunction,
+ PostProcessingFunction ppFunction){
+ if (readFunction!=null){
+ this.readFunction = readFunction ;
+ }
+ this.postProcessingFunction = ppFunction;
+ }
+
+ /**
+ * Segregate post processing function eg: rate from aggregate function,
+ * example: avg, in any order
+ * @param metricName metric name from request
+ * @return @Function
+ */
+ public static Function fromMetricName(String metricName) {
+ // gets postprocessing, and aggregation function
+ // ex. Metric._rate._avg
+ String[] parts = metricName.split(SUFFIX_SEPARATOR);
+
+ ReadFunction readFunction = ReadFunction.VALUE;
+ PostProcessingFunction ppFunction = null;
+
+ if (parts.length <= 1) {
+ return new Function(readFunction, null);
+ }
+ if (parts.length > 3) {
+ throw new IllegalArgumentException("Invalid number of functions specified.");
+ }
+
+ // Parse functions
+ boolean isSuccessful = false; // Best effort
+ for (String part : parts) {
+ if (ReadFunction.isPresent(part)) {
+ readFunction = ReadFunction.getFunction(part);
+ isSuccessful = true;
+ }
+ if (PostProcessingFunction.isPresent(part)) {
+ ppFunction = PostProcessingFunction.getFunction(part);
+ isSuccessful = true;
+ }
+ }
+
+ // Throw exception if parsing failed
+ if (!isSuccessful) {
+ throw new FunctionFormatException("Could not parse provided functions: " +
+ "" + Arrays.asList(parts));
+ }
+
+ return new Function(readFunction, ppFunction);
+ }
+
+ public String getSuffix(){
+ return (postProcessingFunction == null)? readFunction.getSuffix() :
+ postProcessingFunction.getSuffix() + readFunction.getSuffix();
+ }
+
+ public ReadFunction getReadFunction() {
+ return readFunction;
+ }
+
+ @Override
+ public String toString() {
+ return "Function{" +
+ "readFunction=" + readFunction +
+ ", postProcessingFunction=" + postProcessingFunction +
+ '}';
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (!(o instanceof Function)) return false;
+
+ Function function = (Function) o;
+
+ return postProcessingFunction == function.postProcessingFunction
+ && readFunction == function.readFunction;
+
+ }
+
+ @Override
+ public int hashCode() {
+ int result = readFunction.hashCode();
+ result = 31 * result + (postProcessingFunction != null ?
+ postProcessingFunction.hashCode() : 0);
+ return result;
+ }
+
+ public enum PostProcessingFunction {
+ NONE(""),
+ RATE("._rate"),
+ DIFF("._diff");
+
+ PostProcessingFunction(String suffix){
+ this.suffix = suffix;
+ }
+
+ private String suffix = "";
+
+ public String getSuffix(){
+ return suffix;
+ }
+
+ public static boolean isPresent(String functionName) {
+ try {
+ PostProcessingFunction.valueOf(functionName.toUpperCase());
+ } catch (IllegalArgumentException e) {
+ return false;
+ }
+ return true;
+ }
+
+ public static PostProcessingFunction getFunction(String functionName) throws FunctionFormatException {
+ if (functionName == null) {
+ return NONE;
+ }
+
+ try {
+ return PostProcessingFunction.valueOf(functionName.toUpperCase());
+ } catch (IllegalArgumentException e) {
+ throw new FunctionFormatException("Function should be ._rate", e);
+ }
+ }
+ }
+
+ public enum ReadFunction {
+ VALUE(""),
+ AVG("._avg"),
+ MIN("._min"),
+ MAX("._max"),
+ SUM("._sum");
+
+ private final String suffix;
+
+ ReadFunction(String suffix){
+ this.suffix = suffix;
+ }
+
+ public String getSuffix() {
+ return suffix;
+ }
+
+ public static boolean isPresent(String functionName) {
+ try {
+ ReadFunction.valueOf(functionName.toUpperCase());
+ } catch (IllegalArgumentException e) {
+ return false;
+ }
+ return true;
+ }
+
+ public static ReadFunction getFunction(String functionName) throws FunctionFormatException {
+ if (functionName == null) {
+ return VALUE;
+ }
+ try {
+ return ReadFunction.valueOf(functionName.toUpperCase());
+ } catch (IllegalArgumentException e) {
+ throw new FunctionFormatException(
+ "Function should be sum, avg, min, max. Got " + functionName, e);
+ }
+ }
+ }
+
+ public static class FunctionFormatException extends IllegalArgumentException {
+ public FunctionFormatException(String message) {
+ super(message);
+ }
+
+ public FunctionFormatException(String message, Throwable cause) {
+ super(message, cause);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/aggregators/TimelineClusterMetric.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/aggregators/TimelineClusterMetric.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/aggregators/TimelineClusterMetric.java
new file mode 100644
index 0000000..6e81feb
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/aggregators/TimelineClusterMetric.java
@@ -0,0 +1,97 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.metrics.core.timeline.aggregators;
+
+public class TimelineClusterMetric {
+ private String metricName;
+ private String appId;
+ private String instanceId;
+ private long timestamp;
+
+ public TimelineClusterMetric(String metricName, String appId, String instanceId,
+ long timestamp) {
+ this.metricName = metricName;
+ this.appId = appId;
+ this.instanceId = instanceId;
+ this.timestamp = timestamp;
+ }
+
+ public String getMetricName() {
+ return metricName;
+ }
+
+ public String getAppId() {
+ return appId;
+ }
+
+ public String getInstanceId() {
+ return instanceId;
+ }
+
+ public long getTimestamp() {
+ return timestamp;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+
+ TimelineClusterMetric that = (TimelineClusterMetric) o;
+
+ if (timestamp != that.timestamp) return false;
+ if (appId != null ? !appId.equals(that.appId) : that.appId != null)
+ return false;
+ if (instanceId != null ? !instanceId.equals(that.instanceId) : that.instanceId != null)
+ return false;
+ if (!metricName.equals(that.metricName)) return false;
+
+ return true;
+ }
+
+ public boolean equalsExceptTime(TimelineClusterMetric metric) {
+ if (!metricName.equals(metric.metricName)) return false;
+ if (!appId.equals(metric.appId)) return false;
+ if (instanceId != null ? !instanceId.equals(metric.instanceId) : metric.instanceId != null)
+ return false;
+
+ return true;
+ }
+ @Override
+ public int hashCode() {
+ int result = metricName.hashCode();
+ result = 31 * result + (appId != null ? appId.hashCode() : 0);
+ result = 31 * result + (instanceId != null ? instanceId.hashCode() : 0);
+ result = 31 * result + (int) (timestamp ^ (timestamp >>> 32));
+ return result;
+ }
+
+ @Override
+ public String toString() {
+ return "TimelineClusterMetric{" +
+ "metricName='" + metricName + '\'' +
+ ", appId='" + appId + '\'' +
+ ", instanceId='" + instanceId + '\'' +
+ ", timestamp=" + timestamp +
+ '}';
+ }
+
+ public void setTimestamp(long timestamp) {
+ this.timestamp = timestamp;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/aggregators/TimelineMetricAggregator.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/aggregators/TimelineMetricAggregator.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/aggregators/TimelineMetricAggregator.java
new file mode 100644
index 0000000..3698f1b
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/aggregators/TimelineMetricAggregator.java
@@ -0,0 +1,59 @@
+package org.apache.ambari.metrics.core.timeline.aggregators;
+
+import org.apache.ambari.metrics.core.timeline.availability.AggregationTaskRunner.AGGREGATOR_NAME;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+public interface TimelineMetricAggregator extends Runnable {
+ /**
+ * Aggregate metric data within the time bounds.
+ *
+ * @param startTime start time millis
+ * @param endTime end time millis
+ * @return success
+ */
+ boolean doWork(long startTime, long endTime);
+
+ /**
+ * Is aggregator is disabled by configuration.
+ *
+ * @return true/false
+ */
+ boolean isDisabled();
+
+ /**
+ * Return aggregator Interval
+ *
+ * @return Interval in Millis
+ */
+ Long getSleepIntervalMillis();
+
+ /**
+ * Get aggregator name
+ * @return @AGGREGATOR_NAME
+ */
+ AGGREGATOR_NAME getName();
+
+ /**
+ * Known aggregator types
+ */
+ enum AGGREGATOR_TYPE {
+ CLUSTER,
+ HOST
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/aggregators/TimelineMetricAggregatorFactory.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/aggregators/TimelineMetricAggregatorFactory.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/aggregators/TimelineMetricAggregatorFactory.java
new file mode 100644
index 0000000..b395b39
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/aggregators/TimelineMetricAggregatorFactory.java
@@ -0,0 +1,528 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.metrics.core.timeline.aggregators;
+
+import static java.util.concurrent.TimeUnit.SECONDS;
+import static org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguration.CLUSTER_AGGREGATOR_DAILY_CHECKPOINT_CUTOFF_MULTIPLIER;
+import static org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguration.CLUSTER_AGGREGATOR_DAILY_DISABLED;
+import static org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguration.CLUSTER_AGGREGATOR_DAILY_SLEEP_INTERVAL;
+import static org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguration.CLUSTER_AGGREGATOR_HOUR_CHECKPOINT_CUTOFF_MULTIPLIER;
+import static org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguration.CLUSTER_AGGREGATOR_HOUR_DISABLED;
+import static org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguration.CLUSTER_AGGREGATOR_HOUR_SLEEP_INTERVAL;
+import static org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguration.CLUSTER_AGGREGATOR_MINUTE_CHECKPOINT_CUTOFF_MULTIPLIER;
+import static org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguration.CLUSTER_AGGREGATOR_MINUTE_DISABLED;
+import static org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguration.CLUSTER_AGGREGATOR_MINUTE_SLEEP_INTERVAL;
+import static org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguration.CLUSTER_AGGREGATOR_SECOND_CHECKPOINT_CUTOFF_MULTIPLIER;
+import static org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguration.CLUSTER_AGGREGATOR_SECOND_DISABLED;
+import static org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguration.CLUSTER_AGGREGATOR_SECOND_SLEEP_INTERVAL;
+import static org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguration.CLUSTER_AGGREGATOR_TIMESLICE_INTERVAL;
+import static org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguration.DEFAULT_CHECKPOINT_LOCATION;
+import static org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguration.HOST_AGGREGATOR_DAILY_CHECKPOINT_CUTOFF_MULTIPLIER;
+import static org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguration.HOST_AGGREGATOR_DAILY_DISABLED;
+import static org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguration.HOST_AGGREGATOR_DAILY_SLEEP_INTERVAL;
+import static org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguration.HOST_AGGREGATOR_HOUR_CHECKPOINT_CUTOFF_MULTIPLIER;
+import static org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguration.HOST_AGGREGATOR_HOUR_DISABLED;
+import static org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguration.HOST_AGGREGATOR_HOUR_SLEEP_INTERVAL;
+import static org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguration.HOST_AGGREGATOR_MINUTE_CHECKPOINT_CUTOFF_MULTIPLIER;
+import static org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguration.HOST_AGGREGATOR_MINUTE_DISABLED;
+import static org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguration.HOST_AGGREGATOR_MINUTE_SLEEP_INTERVAL;
+import static org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguration.TIMELINE_METRICS_AGGREGATOR_CHECKPOINT_DIR;
+import static org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguration.USE_GROUPBY_AGGREGATOR_QUERIES;
+import static org.apache.ambari.metrics.core.timeline.availability.AggregationTaskRunner.AGGREGATOR_NAME.METRIC_AGGREGATE_DAILY;
+import static org.apache.ambari.metrics.core.timeline.availability.AggregationTaskRunner.AGGREGATOR_NAME.METRIC_AGGREGATE_HOURLY;
+import static org.apache.ambari.metrics.core.timeline.availability.AggregationTaskRunner.AGGREGATOR_NAME.METRIC_AGGREGATE_MINUTE;
+import static org.apache.ambari.metrics.core.timeline.availability.AggregationTaskRunner.AGGREGATOR_NAME.METRIC_AGGREGATE_SECOND;
+import static org.apache.ambari.metrics.core.timeline.availability.AggregationTaskRunner.AGGREGATOR_NAME.METRIC_RECORD_DAILY;
+import static org.apache.ambari.metrics.core.timeline.availability.AggregationTaskRunner.AGGREGATOR_NAME.METRIC_RECORD_HOURLY;
+import static org.apache.ambari.metrics.core.timeline.availability.AggregationTaskRunner.AGGREGATOR_NAME.METRIC_RECORD_MINUTE;
+import static org.apache.ambari.metrics.core.timeline.query.PhoenixTransactSQL.METRICS_AGGREGATE_DAILY_TABLE_NAME;
+import static org.apache.ambari.metrics.core.timeline.query.PhoenixTransactSQL.METRICS_AGGREGATE_HOURLY_TABLE_NAME;
+import static org.apache.ambari.metrics.core.timeline.query.PhoenixTransactSQL.METRICS_AGGREGATE_MINUTE_TABLE_NAME;
+import static org.apache.ambari.metrics.core.timeline.query.PhoenixTransactSQL.METRICS_CLUSTER_AGGREGATE_DAILY_TABLE_NAME;
+import static org.apache.ambari.metrics.core.timeline.query.PhoenixTransactSQL.METRICS_CLUSTER_AGGREGATE_HOURLY_TABLE_NAME;
+import static org.apache.ambari.metrics.core.timeline.query.PhoenixTransactSQL.METRICS_CLUSTER_AGGREGATE_MINUTE_TABLE_NAME;
+import static org.apache.ambari.metrics.core.timeline.query.PhoenixTransactSQL.METRICS_CLUSTER_AGGREGATE_TABLE_NAME;
+import static org.apache.ambari.metrics.core.timeline.query.PhoenixTransactSQL.METRICS_RECORD_TABLE_NAME;
+
+import java.util.concurrent.ConcurrentHashMap;
+
+import org.apache.ambari.metrics.core.timeline.availability.MetricCollectorHAController;
+import org.apache.commons.io.FilenameUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.ambari.metrics.core.timeline.PhoenixHBaseAccessor;
+import org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguration;
+import org.apache.ambari.metrics.core.timeline.TimelineMetricDistributedCache;
+import org.apache.ambari.metrics.core.timeline.discovery.TimelineMetricMetadataManager;
+
+/**
+ * Factory class that knows how to create a aggregator instance using
+ * TimelineMetricConfiguration
+ */
+public class TimelineMetricAggregatorFactory {
+ private static final String HOST_AGGREGATE_MINUTE_CHECKPOINT_FILE =
+ "timeline-metrics-host-aggregator-checkpoint";
+ private static final String HOST_AGGREGATE_HOURLY_CHECKPOINT_FILE =
+ "timeline-metrics-host-aggregator-hourly-checkpoint";
+ private static final String HOST_AGGREGATE_DAILY_CHECKPOINT_FILE =
+ "timeline-metrics-host-aggregator-daily-checkpoint";
+ private static final String CLUSTER_AGGREGATOR_CHECKPOINT_FILE =
+ "timeline-metrics-cluster-aggregator-checkpoint";
+ private static final String CLUSTER_AGGREGATOR_MINUTE_CHECKPOINT_FILE =
+ "timeline-metrics-cluster-aggregator-minute-checkpoint";
+ private static final String CLUSTER_AGGREGATOR_HOURLY_CHECKPOINT_FILE =
+ "timeline-metrics-cluster-aggregator-hourly-checkpoint";
+ private static final String CLUSTER_AGGREGATOR_DAILY_CHECKPOINT_FILE =
+ "timeline-metrics-cluster-aggregator-daily-checkpoint";
+
+ private static boolean useGroupByAggregator(Configuration metricsConf) {
+ return Boolean.parseBoolean(metricsConf.get(USE_GROUPBY_AGGREGATOR_QUERIES, "true"));
+ }
+
+ /**
+ * Minute based aggregation for hosts.
+ * Interval : 5 mins
+ */
+ public static TimelineMetricAggregator createTimelineMetricAggregatorMinute
+ (PhoenixHBaseAccessor hBaseAccessor, Configuration metricsConf,
+ TimelineMetricMetadataManager metadataManager,
+ MetricCollectorHAController haController) {
+
+ String checkpointDir = metricsConf.get(
+ TIMELINE_METRICS_AGGREGATOR_CHECKPOINT_DIR, DEFAULT_CHECKPOINT_LOCATION);
+ String checkpointLocation = FilenameUtils.concat(checkpointDir,
+ HOST_AGGREGATE_MINUTE_CHECKPOINT_FILE);
+ long sleepIntervalMillis = SECONDS.toMillis(metricsConf.getLong
+ (HOST_AGGREGATOR_MINUTE_SLEEP_INTERVAL, 300l)); // 5 mins
+
+ int checkpointCutOffMultiplier = metricsConf.getInt
+ (HOST_AGGREGATOR_MINUTE_CHECKPOINT_CUTOFF_MULTIPLIER, 3);
+ String hostAggregatorDisabledParam = HOST_AGGREGATOR_MINUTE_DISABLED;
+
+ String inputTableName = METRICS_RECORD_TABLE_NAME;
+ String outputTableName = METRICS_AGGREGATE_MINUTE_TABLE_NAME;
+
+ if (useGroupByAggregator(metricsConf)) {
+ return new org.apache.ambari.metrics.core.timeline.aggregators.v2.TimelineMetricHostAggregator(
+ METRIC_RECORD_MINUTE,
+ hBaseAccessor, metricsConf,
+ checkpointLocation,
+ sleepIntervalMillis,
+ checkpointCutOffMultiplier,
+ hostAggregatorDisabledParam,
+ inputTableName,
+ outputTableName,
+ 120000l,
+ haController
+ );
+ }
+
+ return new TimelineMetricHostAggregator(
+ METRIC_RECORD_MINUTE,
+ metadataManager,
+ hBaseAccessor, metricsConf,
+ checkpointLocation,
+ sleepIntervalMillis,
+ checkpointCutOffMultiplier,
+ hostAggregatorDisabledParam,
+ inputTableName,
+ outputTableName,
+ 120000l,
+ haController);
+ }
+
+ /**
+ * Hourly aggregation for hosts.
+ * Interval : 1 hour
+ */
+ public static TimelineMetricAggregator createTimelineMetricAggregatorHourly
+ (PhoenixHBaseAccessor hBaseAccessor, Configuration metricsConf,
+ TimelineMetricMetadataManager metadataManager,
+ MetricCollectorHAController haController) {
+
+ String checkpointDir = metricsConf.get(
+ TIMELINE_METRICS_AGGREGATOR_CHECKPOINT_DIR, DEFAULT_CHECKPOINT_LOCATION);
+ String checkpointLocation = FilenameUtils.concat(checkpointDir,
+ HOST_AGGREGATE_HOURLY_CHECKPOINT_FILE);
+ long sleepIntervalMillis = SECONDS.toMillis(metricsConf.getLong
+ (HOST_AGGREGATOR_HOUR_SLEEP_INTERVAL, 3600l));
+
+ int checkpointCutOffMultiplier = metricsConf.getInt
+ (HOST_AGGREGATOR_HOUR_CHECKPOINT_CUTOFF_MULTIPLIER, 2);
+ String hostAggregatorDisabledParam = HOST_AGGREGATOR_HOUR_DISABLED;
+
+ String inputTableName = METRICS_AGGREGATE_MINUTE_TABLE_NAME;
+ String outputTableName = METRICS_AGGREGATE_HOURLY_TABLE_NAME;
+
+ if (useGroupByAggregator(metricsConf)) {
+ return new org.apache.ambari.metrics.core.timeline.aggregators.v2.TimelineMetricHostAggregator(
+ METRIC_RECORD_HOURLY,
+ hBaseAccessor, metricsConf,
+ checkpointLocation,
+ sleepIntervalMillis,
+ checkpointCutOffMultiplier,
+ hostAggregatorDisabledParam,
+ inputTableName,
+ outputTableName,
+ 3600000l,
+ haController
+ );
+ }
+
+ return new TimelineMetricHostAggregator(
+ METRIC_RECORD_HOURLY,
+ metadataManager,
+ hBaseAccessor, metricsConf,
+ checkpointLocation,
+ sleepIntervalMillis,
+ checkpointCutOffMultiplier,
+ hostAggregatorDisabledParam,
+ inputTableName,
+ outputTableName,
+ 3600000l,
+ haController);
+ }
+
+ /**
+ * Daily aggregation for hosts.
+ * Interval : 1 day
+ */
+ public static TimelineMetricAggregator createTimelineMetricAggregatorDaily
+ (PhoenixHBaseAccessor hBaseAccessor, Configuration metricsConf,
+ TimelineMetricMetadataManager metadataManager,
+ MetricCollectorHAController haController) {
+
+ String checkpointDir = metricsConf.get(
+ TIMELINE_METRICS_AGGREGATOR_CHECKPOINT_DIR, DEFAULT_CHECKPOINT_LOCATION);
+ String checkpointLocation = FilenameUtils.concat(checkpointDir,
+ HOST_AGGREGATE_DAILY_CHECKPOINT_FILE);
+ long sleepIntervalMillis = SECONDS.toMillis(metricsConf.getLong
+ (HOST_AGGREGATOR_DAILY_SLEEP_INTERVAL, 86400l));
+
+ int checkpointCutOffMultiplier = metricsConf.getInt
+ (HOST_AGGREGATOR_DAILY_CHECKPOINT_CUTOFF_MULTIPLIER, 1);
+ String hostAggregatorDisabledParam = HOST_AGGREGATOR_DAILY_DISABLED;
+
+ String inputTableName = METRICS_AGGREGATE_HOURLY_TABLE_NAME;
+ String outputTableName = METRICS_AGGREGATE_DAILY_TABLE_NAME;
+
+ if (useGroupByAggregator(metricsConf)) {
+ return new org.apache.ambari.metrics.core.timeline.aggregators.v2.TimelineMetricHostAggregator(
+ METRIC_RECORD_DAILY,
+ hBaseAccessor, metricsConf,
+ checkpointLocation,
+ sleepIntervalMillis,
+ checkpointCutOffMultiplier,
+ hostAggregatorDisabledParam,
+ inputTableName,
+ outputTableName,
+ 3600000l,
+ haController
+ );
+ }
+
+ return new TimelineMetricHostAggregator(
+ METRIC_RECORD_DAILY,
+ metadataManager,
+ hBaseAccessor, metricsConf,
+ checkpointLocation,
+ sleepIntervalMillis,
+ checkpointCutOffMultiplier,
+ hostAggregatorDisabledParam,
+ inputTableName,
+ outputTableName,
+ 3600000l,
+ haController);
+ }
+
+ /**
+ * Second aggregation for cluster.
+ * Interval : 2 mins
+ * Timeslice : 30 sec
+ */
+ public static TimelineMetricAggregator createTimelineClusterAggregatorSecond(
+ PhoenixHBaseAccessor hBaseAccessor, Configuration metricsConf,
+ TimelineMetricMetadataManager metadataManager,
+ MetricCollectorHAController haController,
+ TimelineMetricDistributedCache distributedCache) {
+
+ String checkpointDir = metricsConf.get(
+ TIMELINE_METRICS_AGGREGATOR_CHECKPOINT_DIR, DEFAULT_CHECKPOINT_LOCATION);
+
+ String checkpointLocation = FilenameUtils.concat(checkpointDir,
+ CLUSTER_AGGREGATOR_CHECKPOINT_FILE);
+
+ long sleepIntervalMillis = SECONDS.toMillis(metricsConf.getLong
+ (CLUSTER_AGGREGATOR_SECOND_SLEEP_INTERVAL, 120l));
+
+ long timeSliceIntervalMillis = SECONDS.toMillis(metricsConf.getInt
+ (CLUSTER_AGGREGATOR_TIMESLICE_INTERVAL, 30));
+
+ int checkpointCutOffMultiplier =
+ metricsConf.getInt(CLUSTER_AGGREGATOR_SECOND_CHECKPOINT_CUTOFF_MULTIPLIER, 2);
+
+ String outputTableName = METRICS_CLUSTER_AGGREGATE_TABLE_NAME;
+ String aggregatorDisabledParam = CLUSTER_AGGREGATOR_SECOND_DISABLED;
+
+ // Second based aggregation have added responsibility of time slicing
+ if (TimelineMetricConfiguration.getInstance().isCollectorInMemoryAggregationEnabled()) {
+ return new TimelineMetricClusterAggregatorSecondWithCacheSource(
+ METRIC_AGGREGATE_SECOND,
+ metadataManager,
+ hBaseAccessor, metricsConf,
+ checkpointLocation,
+ sleepIntervalMillis,
+ checkpointCutOffMultiplier,
+ aggregatorDisabledParam,
+ null,
+ outputTableName,
+ 120000l,
+ timeSliceIntervalMillis,
+ haController,
+ distributedCache
+ );
+ }
+
+ String inputTableName = METRICS_RECORD_TABLE_NAME;
+ return new TimelineMetricClusterAggregatorSecond(
+ METRIC_AGGREGATE_SECOND,
+ metadataManager,
+ hBaseAccessor, metricsConf,
+ checkpointLocation,
+ sleepIntervalMillis,
+ checkpointCutOffMultiplier,
+ aggregatorDisabledParam,
+ inputTableName,
+ outputTableName,
+ 120000l,
+ timeSliceIntervalMillis,
+ haController
+ );
+ }
+
+ /**
+ * Minute aggregation for cluster.
+ * Interval : 5 mins
+ */
+ public static TimelineMetricAggregator createTimelineClusterAggregatorMinute(
+ PhoenixHBaseAccessor hBaseAccessor, Configuration metricsConf,
+ TimelineMetricMetadataManager metadataManager,
+ MetricCollectorHAController haController) {
+
+ String checkpointDir = metricsConf.get(
+ TIMELINE_METRICS_AGGREGATOR_CHECKPOINT_DIR, DEFAULT_CHECKPOINT_LOCATION);
+
+ String checkpointLocation = FilenameUtils.concat(checkpointDir,
+ CLUSTER_AGGREGATOR_MINUTE_CHECKPOINT_FILE);
+
+ long sleepIntervalMillis = SECONDS.toMillis(metricsConf.getLong
+ (CLUSTER_AGGREGATOR_MINUTE_SLEEP_INTERVAL, 300l));
+
+ int checkpointCutOffMultiplier = metricsConf.getInt
+ (CLUSTER_AGGREGATOR_MINUTE_CHECKPOINT_CUTOFF_MULTIPLIER, 2);
+
+ String inputTableName = METRICS_CLUSTER_AGGREGATE_TABLE_NAME;
+ String outputTableName = METRICS_CLUSTER_AGGREGATE_MINUTE_TABLE_NAME;
+ String aggregatorDisabledParam = CLUSTER_AGGREGATOR_MINUTE_DISABLED;
+
+ if (useGroupByAggregator(metricsConf)) {
+ return new org.apache.ambari.metrics.core.timeline.aggregators.v2.TimelineMetricClusterAggregator(
+ METRIC_AGGREGATE_MINUTE,
+ hBaseAccessor, metricsConf,
+ checkpointLocation,
+ sleepIntervalMillis,
+ checkpointCutOffMultiplier,
+ aggregatorDisabledParam,
+ inputTableName,
+ outputTableName,
+ 120000l,
+ haController
+ );
+ }
+
+ return new TimelineMetricClusterAggregator(
+ METRIC_AGGREGATE_MINUTE,
+ metadataManager,
+ hBaseAccessor, metricsConf,
+ checkpointLocation,
+ sleepIntervalMillis,
+ checkpointCutOffMultiplier,
+ aggregatorDisabledParam,
+ inputTableName,
+ outputTableName,
+ 120000l,
+ haController
+ );
+ }
+
+ /**
+ * Hourly aggregation for cluster.
+ * Interval : 1 hour
+ */
+ public static TimelineMetricAggregator createTimelineClusterAggregatorHourly(
+ PhoenixHBaseAccessor hBaseAccessor, Configuration metricsConf,
+ TimelineMetricMetadataManager metadataManager,
+ MetricCollectorHAController haController) {
+
+ String checkpointDir = metricsConf.get(
+ TIMELINE_METRICS_AGGREGATOR_CHECKPOINT_DIR, DEFAULT_CHECKPOINT_LOCATION);
+
+ String checkpointLocation = FilenameUtils.concat(checkpointDir,
+ CLUSTER_AGGREGATOR_HOURLY_CHECKPOINT_FILE);
+
+ long sleepIntervalMillis = SECONDS.toMillis(metricsConf.getLong
+ (CLUSTER_AGGREGATOR_HOUR_SLEEP_INTERVAL, 3600l));
+
+ int checkpointCutOffMultiplier = metricsConf.getInt
+ (CLUSTER_AGGREGATOR_HOUR_CHECKPOINT_CUTOFF_MULTIPLIER, 2);
+
+ String inputTableName = METRICS_CLUSTER_AGGREGATE_TABLE_NAME;
+ String outputTableName = METRICS_CLUSTER_AGGREGATE_HOURLY_TABLE_NAME;
+ String aggregatorDisabledParam = CLUSTER_AGGREGATOR_HOUR_DISABLED;
+
+ if (useGroupByAggregator(metricsConf)) {
+ return new org.apache.ambari.metrics.core.timeline.aggregators.v2.TimelineMetricClusterAggregator(
+ METRIC_AGGREGATE_HOURLY,
+ hBaseAccessor, metricsConf,
+ checkpointLocation,
+ sleepIntervalMillis,
+ checkpointCutOffMultiplier,
+ aggregatorDisabledParam,
+ inputTableName,
+ outputTableName,
+ 120000l,
+ haController
+ );
+ }
+
+ return new TimelineMetricClusterAggregator(
+ METRIC_AGGREGATE_HOURLY,
+ metadataManager,
+ hBaseAccessor, metricsConf,
+ checkpointLocation,
+ sleepIntervalMillis,
+ checkpointCutOffMultiplier,
+ aggregatorDisabledParam,
+ inputTableName,
+ outputTableName,
+ 120000l,
+ haController
+ );
+ }
+
+ /**
+ * Daily aggregation for cluster.
+ * Interval : 1 day
+ */
+ public static TimelineMetricAggregator createTimelineClusterAggregatorDaily(
+ PhoenixHBaseAccessor hBaseAccessor, Configuration metricsConf,
+ TimelineMetricMetadataManager metadataManager,
+ MetricCollectorHAController haController) {
+
+ String checkpointDir = metricsConf.get(
+ TIMELINE_METRICS_AGGREGATOR_CHECKPOINT_DIR, DEFAULT_CHECKPOINT_LOCATION);
+
+ String checkpointLocation = FilenameUtils.concat(checkpointDir,
+ CLUSTER_AGGREGATOR_DAILY_CHECKPOINT_FILE);
+
+ long sleepIntervalMillis = SECONDS.toMillis(metricsConf.getLong
+ (CLUSTER_AGGREGATOR_DAILY_SLEEP_INTERVAL, 86400l));
+
+ int checkpointCutOffMultiplier = metricsConf.getInt
+ (CLUSTER_AGGREGATOR_DAILY_CHECKPOINT_CUTOFF_MULTIPLIER, 1);
+
+ String inputTableName = METRICS_CLUSTER_AGGREGATE_HOURLY_TABLE_NAME;
+ String outputTableName = METRICS_CLUSTER_AGGREGATE_DAILY_TABLE_NAME;
+ String aggregatorDisabledParam = CLUSTER_AGGREGATOR_DAILY_DISABLED;
+
+ if (useGroupByAggregator(metricsConf)) {
+ return new org.apache.ambari.metrics.core.timeline.aggregators.v2.TimelineMetricClusterAggregator(
+ METRIC_AGGREGATE_DAILY,
+ hBaseAccessor, metricsConf,
+ checkpointLocation,
+ sleepIntervalMillis,
+ checkpointCutOffMultiplier,
+ aggregatorDisabledParam,
+ inputTableName,
+ outputTableName,
+ 120000l,
+ haController
+ );
+ }
+
+ return new TimelineMetricClusterAggregator(
+ METRIC_AGGREGATE_DAILY,
+ metadataManager,
+ hBaseAccessor, metricsConf,
+ checkpointLocation,
+ sleepIntervalMillis,
+ checkpointCutOffMultiplier,
+ aggregatorDisabledParam,
+ inputTableName,
+ outputTableName,
+ 120000l,
+ haController
+ );
+ }
+
+ public static TimelineMetricAggregator createFilteringTimelineMetricAggregatorMinute(PhoenixHBaseAccessor hBaseAccessor, Configuration metricsConf, TimelineMetricMetadataManager metricMetadataManager, MetricCollectorHAController haController, ConcurrentHashMap<String, Long> postedAggregatedMap) {
+ String checkpointDir = metricsConf.get(
+ TIMELINE_METRICS_AGGREGATOR_CHECKPOINT_DIR, DEFAULT_CHECKPOINT_LOCATION);
+ String checkpointLocation = FilenameUtils.concat(checkpointDir,
+ HOST_AGGREGATE_MINUTE_CHECKPOINT_FILE);
+ long sleepIntervalMillis = SECONDS.toMillis(metricsConf.getLong
+ (HOST_AGGREGATOR_MINUTE_SLEEP_INTERVAL, 300l)); // 5 mins
+
+ int checkpointCutOffMultiplier = metricsConf.getInt
+ (HOST_AGGREGATOR_MINUTE_CHECKPOINT_CUTOFF_MULTIPLIER, 3);
+ String hostAggregatorDisabledParam = HOST_AGGREGATOR_MINUTE_DISABLED;
+
+ String inputTableName = METRICS_RECORD_TABLE_NAME;
+ String outputTableName = METRICS_AGGREGATE_MINUTE_TABLE_NAME;
+
+ if (useGroupByAggregator(metricsConf)) {
+ return new org.apache.ambari.metrics.core.timeline.aggregators.v2.TimelineMetricFilteringHostAggregator(
+ METRIC_RECORD_MINUTE,
+ metricMetadataManager,
+ hBaseAccessor, metricsConf,
+ checkpointLocation,
+ sleepIntervalMillis,
+ checkpointCutOffMultiplier,
+ hostAggregatorDisabledParam,
+ inputTableName,
+ outputTableName,
+ 120000l,
+ haController,
+ postedAggregatedMap
+ );
+ }
+
+ return new TimelineMetricFilteringHostAggregator(
+ METRIC_RECORD_MINUTE,
+ metricMetadataManager,
+ hBaseAccessor, metricsConf,
+ checkpointLocation,
+ sleepIntervalMillis,
+ checkpointCutOffMultiplier,
+ hostAggregatorDisabledParam,
+ inputTableName,
+ outputTableName,
+ 120000l,
+ haController,
+ postedAggregatedMap);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/aggregators/TimelineMetricAppAggregator.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/aggregators/TimelineMetricAppAggregator.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/aggregators/TimelineMetricAppAggregator.java
new file mode 100644
index 0000000..190ad9a
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/aggregators/TimelineMetricAppAggregator.java
@@ -0,0 +1,187 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.metrics.core.timeline.aggregators;
+
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+
+import org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguration;
+import org.apache.ambari.metrics.core.timeline.TimelineMetricsFilter;
+import org.apache.ambari.metrics.core.timeline.discovery.TimelineMetricHostMetadata;
+import org.apache.ambari.metrics.core.timeline.discovery.TimelineMetricMetadataKey;
+import org.apache.ambari.metrics.core.timeline.discovery.TimelineMetricMetadataManager;
+import org.apache.commons.lang.StringUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.metrics2.sink.timeline.MetricClusterAggregate;
+import org.apache.hadoop.metrics2.sink.timeline.TimelineMetricMetadata;
+
+/**
+ * Aggregator responsible for providing app level host aggregates. This task
+ * is accomplished without doing a round trip to storage, rather
+ * TimelineMetricClusterAggregators are responsible for lifecycle of
+ * @TimelineMetricAppAggregator and provide the raw data to aggregate.
+ */
+public class TimelineMetricAppAggregator {
+ private static final Log LOG = LogFactory.getLog(TimelineMetricAppAggregator.class);
+ // Lookup to check candidacy of an app
+ private final List<String> appIdsToAggregate;
+ private final Map<String, TimelineMetricHostMetadata> hostMetadata;
+ Map<TimelineClusterMetric, MetricClusterAggregate> aggregateClusterMetrics = new HashMap<>();
+ TimelineMetricMetadataManager metadataManagerInstance;
+
+ public TimelineMetricAppAggregator(TimelineMetricMetadataManager metadataManager,
+ Configuration metricsConf) {
+ appIdsToAggregate = getAppIdsForHostAggregation(metricsConf);
+ hostMetadata = metadataManager.getHostedAppsCache();
+ metadataManagerInstance = metadataManager;
+ LOG.info("AppIds configured for aggregation: " + appIdsToAggregate);
+ }
+
+ /**
+ * Lifecycle method to initialize aggregation cycle.
+ */
+ public void init() {
+ LOG.debug("Initializing aggregation cycle.");
+ aggregateClusterMetrics = new HashMap<>();
+ }
+
+ /**
+ * Lifecycle method to indicate end of aggregation cycle.
+ */
+ public void cleanup() {
+ LOG.debug("Cleanup aggregated data.");
+ aggregateClusterMetrics = null;
+ }
+
+ /**
+ * Calculate aggregates if the clusterMetric is a Host metric for recorded
+ * apps that are housed by this host.
+ *
+ * @param clusterMetric @TimelineClusterMetric Host / App metric
+ * @param hostname This is the hostname from which this clusterMetric originated.
+ * @param metricValue The metric value for this metric.
+ */
+ public void processTimelineClusterMetric(TimelineClusterMetric clusterMetric,
+ String hostname, Double metricValue) {
+
+ String appId = clusterMetric.getAppId();
+ if (appId == null) {
+ return; // No real use case except tests
+ }
+
+ // If metric is a host metric and host has apps on it
+ if (appId.equalsIgnoreCase(TimelineMetricConfiguration.HOST_APP_ID)) {
+ // Candidate metric, update app aggregates
+ if (hostMetadata.containsKey(hostname)) {
+ updateAppAggregatesFromHostMetric(clusterMetric, hostname, metricValue);
+ }
+ } else {
+ // Build the hostedapps map if not a host metric
+ // Check app candidacy for host aggregation
+ if (appIdsToAggregate.contains(appId)) {
+ TimelineMetricHostMetadata timelineMetricHostMetadata = hostMetadata.get(hostname);
+ ConcurrentHashMap<String, String> appIds;
+ if (timelineMetricHostMetadata == null) {
+ appIds = new ConcurrentHashMap<>();
+ hostMetadata.put(hostname, new TimelineMetricHostMetadata(appIds));
+ } else {
+ appIds = timelineMetricHostMetadata.getHostedApps();
+ }
+ if (!appIds.containsKey(appId)) {
+ appIds.put(appId, appId);
+ LOG.info("Adding appId to hosted apps: appId = " +
+ clusterMetric.getAppId() + ", hostname = " + hostname);
+ }
+ }
+ }
+ }
+
+ /**
+ * Build a cluster app metric from a host metric
+ */
+ private void updateAppAggregatesFromHostMetric(TimelineClusterMetric clusterMetric,
+ String hostname, Double metricValue) {
+
+ if (aggregateClusterMetrics == null) {
+ LOG.error("Aggregation requested without init call.");
+ return;
+ }
+
+ TimelineMetricMetadataKey appKey = new TimelineMetricMetadataKey(clusterMetric.getMetricName(), TimelineMetricConfiguration.HOST_APP_ID, clusterMetric.getInstanceId());
+ ConcurrentHashMap<String, String> apps = hostMetadata.get(hostname).getHostedApps();
+ for (String appId : apps.keySet()) {
+ if (appIdsToAggregate.contains(appId)) {
+
+ appKey.setAppId(appId);
+ TimelineMetricMetadata appMetadata = metadataManagerInstance.getMetadataCacheValue(appKey);
+ if (appMetadata == null) {
+ TimelineMetricMetadataKey key = new TimelineMetricMetadataKey(clusterMetric.getMetricName(), TimelineMetricConfiguration.HOST_APP_ID, clusterMetric.getInstanceId());
+ TimelineMetricMetadata hostMetricMetadata = metadataManagerInstance.getMetadataCacheValue(key);
+
+ if (hostMetricMetadata != null) {
+ TimelineMetricMetadata timelineMetricMetadata = new TimelineMetricMetadata(clusterMetric.getMetricName(),
+ appId, clusterMetric.getInstanceId(), hostMetricMetadata.getUnits(), hostMetricMetadata.getType(), hostMetricMetadata.getSeriesStartTime(),
+ hostMetricMetadata.isSupportsAggregates(), TimelineMetricsFilter.acceptMetric(clusterMetric.getMetricName(), appId));
+ metadataManagerInstance.putIfModifiedTimelineMetricMetadata(timelineMetricMetadata);
+ }
+ }
+
+ // Add a new cluster aggregate metric if none exists
+ TimelineClusterMetric appTimelineClusterMetric =
+ new TimelineClusterMetric(clusterMetric.getMetricName(),
+ appId,
+ clusterMetric.getInstanceId(),
+ clusterMetric.getTimestamp());
+
+ MetricClusterAggregate clusterAggregate = aggregateClusterMetrics.get(appTimelineClusterMetric);
+
+ if (clusterAggregate == null) {
+ clusterAggregate = new MetricClusterAggregate(metricValue, 1, null, metricValue, metricValue);
+ aggregateClusterMetrics.put(appTimelineClusterMetric, clusterAggregate);
+ } else {
+ clusterAggregate.updateSum(metricValue);
+ clusterAggregate.updateNumberOfHosts(1);
+ clusterAggregate.updateMax(metricValue);
+ clusterAggregate.updateMin(metricValue);
+ }
+ }
+
+ }
+ }
+
+ /**
+ * Return current copy of aggregated data.
+ */
+ public Map<TimelineClusterMetric, MetricClusterAggregate> getAggregateClusterMetrics() {
+ return aggregateClusterMetrics;
+ }
+
+ private List<String> getAppIdsForHostAggregation(Configuration metricsConf) {
+ String appIds = metricsConf.get(TimelineMetricConfiguration.CLUSTER_AGGREGATOR_APP_IDS);
+ if (!StringUtils.isEmpty(appIds)) {
+ return Arrays.asList(StringUtils.stripAll(appIds.split(",")));
+ }
+ return Collections.emptyList();
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/aggregators/TimelineMetricClusterAggregator.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/aggregators/TimelineMetricClusterAggregator.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/aggregators/TimelineMetricClusterAggregator.java
new file mode 100644
index 0000000..2ea5309
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/aggregators/TimelineMetricClusterAggregator.java
@@ -0,0 +1,150 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.metrics.core.timeline.aggregators;
+
+import static org.apache.ambari.metrics.core.timeline.query.PhoenixTransactSQL.GET_CLUSTER_AGGREGATE_SQL;
+import static org.apache.ambari.metrics.core.timeline.query.PhoenixTransactSQL.GET_CLUSTER_AGGREGATE_TIME_SQL;
+import static org.apache.ambari.metrics.core.timeline.query.PhoenixTransactSQL.METRICS_CLUSTER_AGGREGATE_TABLE_NAME;
+
+import java.io.IOException;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.ambari.metrics.core.timeline.availability.MetricCollectorHAController;
+import org.apache.ambari.metrics.core.timeline.query.DefaultCondition;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.metrics2.sink.timeline.MetricClusterAggregate;
+import org.apache.hadoop.metrics2.sink.timeline.MetricHostAggregate;
+import org.apache.ambari.metrics.core.timeline.PhoenixHBaseAccessor;
+import org.apache.ambari.metrics.core.timeline.availability.AggregationTaskRunner.AGGREGATOR_NAME;
+import org.apache.ambari.metrics.core.timeline.discovery.TimelineMetricMetadataManager;
+import org.apache.ambari.metrics.core.timeline.query.Condition;
+
+public class TimelineMetricClusterAggregator extends AbstractTimelineAggregator {
+ private final TimelineMetricReadHelper readHelper;
+ private final boolean isClusterPrecisionInputTable;
+
+ public TimelineMetricClusterAggregator(AGGREGATOR_NAME aggregatorName,
+ TimelineMetricMetadataManager metricMetadataManager,
+ PhoenixHBaseAccessor hBaseAccessor,
+ Configuration metricsConf,
+ String checkpointLocation,
+ Long sleepIntervalMillis,
+ Integer checkpointCutOffMultiplier,
+ String hostAggregatorDisabledParam,
+ String inputTableName,
+ String outputTableName,
+ Long nativeTimeRangeDelay,
+ MetricCollectorHAController haController) {
+ super(aggregatorName, hBaseAccessor, metricsConf, checkpointLocation,
+ sleepIntervalMillis, checkpointCutOffMultiplier,
+ hostAggregatorDisabledParam, inputTableName, outputTableName,
+ nativeTimeRangeDelay, haController);
+ isClusterPrecisionInputTable = inputTableName.equals(METRICS_CLUSTER_AGGREGATE_TABLE_NAME);
+ readHelper = new TimelineMetricReadHelper(metricMetadataManager, true);
+ }
+
+ @Override
+ protected Condition prepareMetricQueryCondition(long startTime, long endTime) {
+ Condition condition = new DefaultCondition(null, null, null, null, startTime,
+ endTime, null, null, true);
+ condition.setNoLimit();
+ condition.setFetchSize(resultsetFetchSize);
+ String sqlStr = String.format(GET_CLUSTER_AGGREGATE_TIME_SQL, tableName);
+ // HOST_COUNT vs METRIC_COUNT
+ if (isClusterPrecisionInputTable) {
+ sqlStr = String.format(GET_CLUSTER_AGGREGATE_SQL, tableName);
+ }
+
+ condition.setStatement(sqlStr);
+ condition.addOrderByColumn("UUID");
+ condition.addOrderByColumn("SERVER_TIME");
+ return condition;
+ }
+
+ @Override
+ protected void aggregate(ResultSet rs, long startTime, long endTime) throws IOException, SQLException {
+ Map<TimelineClusterMetric, MetricHostAggregate> hostAggregateMap = aggregateMetricsFromResultSet(rs, endTime);
+
+ LOG.info("Saving " + hostAggregateMap.size() + " metric aggregates.");
+ hBaseAccessor.saveClusterAggregateRecordsSecond(hostAggregateMap, outputTableName);
+ }
+
+ private Map<TimelineClusterMetric, MetricHostAggregate> aggregateMetricsFromResultSet(ResultSet rs, long endTime)
+ throws IOException, SQLException {
+
+ TimelineClusterMetric existingMetric = null;
+ MetricHostAggregate hostAggregate = null;
+ Map<TimelineClusterMetric, MetricHostAggregate> hostAggregateMap =
+ new HashMap<TimelineClusterMetric, MetricHostAggregate>();
+ int perMetricCount = 0;
+
+ while (rs.next()) {
+ TimelineClusterMetric currentMetric = readHelper.fromResultSet(rs);
+
+ MetricClusterAggregate currentHostAggregate =
+ isClusterPrecisionInputTable ?
+ readHelper.getMetricClusterAggregateFromResultSet(rs) :
+ readHelper.getMetricClusterTimeAggregateFromResultSet(rs);
+
+ if (existingMetric == null) {
+ // First row
+ existingMetric = currentMetric;
+ currentMetric.setTimestamp(endTime);
+ hostAggregate = new MetricHostAggregate();
+ hostAggregateMap.put(currentMetric, hostAggregate);
+ perMetricCount++;
+ }
+
+ if (existingMetric.equalsExceptTime(currentMetric)) {
+ // Recalculate totals with current metric
+ updateAggregatesFromHost(hostAggregate, currentHostAggregate);
+ perMetricCount++;
+ } else {
+ // Switched over to a new metric - save new metric
+
+ hostAggregate.setSum(hostAggregate.getSum() / (perMetricCount - 1));
+ hostAggregate.setNumberOfSamples(Math.round((float)hostAggregate.getNumberOfSamples() / (float)(perMetricCount - 1)));
+ perMetricCount = 1;
+
+ hostAggregate = new MetricHostAggregate();
+ currentMetric.setTimestamp(endTime);
+ updateAggregatesFromHost(hostAggregate, currentHostAggregate);
+ hostAggregateMap.put(currentMetric, hostAggregate);
+ existingMetric = currentMetric;
+ }
+
+ }
+
+ if (existingMetric != null) {
+ hostAggregate.setSum(hostAggregate.getSum() / perMetricCount);
+ hostAggregate.setNumberOfSamples(Math.round((float)hostAggregate.getNumberOfSamples() / (float)perMetricCount));
+ }
+
+ return hostAggregateMap;
+ }
+
+ private void updateAggregatesFromHost(MetricHostAggregate agg, MetricClusterAggregate currentClusterAggregate) {
+ agg.updateMax(currentClusterAggregate.getMax());
+ agg.updateMin(currentClusterAggregate.getMin());
+ agg.updateSum(currentClusterAggregate.getSum());
+ agg.updateNumberOfSamples(currentClusterAggregate.getNumberOfHosts());
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/aggregators/TimelineMetricClusterAggregatorSecond.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/aggregators/TimelineMetricClusterAggregatorSecond.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/aggregators/TimelineMetricClusterAggregatorSecond.java
new file mode 100644
index 0000000..46c82d6
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/aggregators/TimelineMetricClusterAggregatorSecond.java
@@ -0,0 +1,263 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.metrics.core.timeline.aggregators;
+
+
+import static org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguration.SERVER_SIDE_TIMESIFT_ADJUSTMENT;
+import static org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguration.TIMELINE_METRICS_CLUSTER_AGGREGATOR_INTERPOLATION_ENABLED;
+import static org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguration.TIMELINE_METRIC_AGGREGATION_SQL_FILTERS;
+import static org.apache.ambari.metrics.core.timeline.aggregators.AggregatorUtils.getTimeSlices;
+import static org.apache.ambari.metrics.core.timeline.aggregators.AggregatorUtils.sliceFromTimelineMetric;
+import static org.apache.ambari.metrics.core.timeline.query.PhoenixTransactSQL.GET_METRIC_SQL;
+import static org.apache.ambari.metrics.core.timeline.query.PhoenixTransactSQL.METRICS_RECORD_TABLE_NAME;
+
+import java.io.IOException;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang.mutable.MutableInt;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.metrics2.sink.timeline.MetricClusterAggregate;
+import org.apache.hadoop.metrics2.sink.timeline.TimelineMetric;
+import org.apache.hadoop.metrics2.sink.timeline.TimelineMetricMetadata;
+import org.apache.ambari.metrics.core.timeline.PhoenixHBaseAccessor;
+import org.apache.ambari.metrics.core.timeline.availability.AggregationTaskRunner.AGGREGATOR_NAME;
+import org.apache.ambari.metrics.core.timeline.availability.MetricCollectorHAController;
+import org.apache.ambari.metrics.core.timeline.discovery.TimelineMetricMetadataKey;
+import org.apache.ambari.metrics.core.timeline.discovery.TimelineMetricMetadataManager;
+import org.apache.ambari.metrics.core.timeline.query.Condition;
+import org.apache.ambari.metrics.core.timeline.query.DefaultCondition;
+
+/**
+ * Aggregates a metric across all hosts in the cluster. Reads metrics from
+ * the precision table and saves into the aggregate.
+ */
+public class TimelineMetricClusterAggregatorSecond extends AbstractTimelineAggregator {
+ public Long timeSliceIntervalMillis;
+ private TimelineMetricReadHelper timelineMetricReadHelper;
+ // Aggregator to perform app-level aggregates for host metrics
+ private final TimelineMetricAppAggregator appAggregator;
+ // 1 minute client side buffering adjustment
+ protected final Long serverTimeShiftAdjustment;
+ protected final boolean interpolationEnabled;
+ private TimelineMetricMetadataManager metadataManagerInstance;
+ private String skipAggrPatternStrings;
+ private final static String liveHostsMetricName = "live_hosts";
+
+ public TimelineMetricClusterAggregatorSecond(AGGREGATOR_NAME aggregatorName,
+ TimelineMetricMetadataManager metadataManager,
+ PhoenixHBaseAccessor hBaseAccessor,
+ Configuration metricsConf,
+ String checkpointLocation,
+ Long sleepIntervalMillis,
+ Integer checkpointCutOffMultiplier,
+ String aggregatorDisabledParam,
+ String tableName,
+ String outputTableName,
+ Long nativeTimeRangeDelay,
+ Long timeSliceInterval,
+ MetricCollectorHAController haController) {
+ super(aggregatorName, hBaseAccessor, metricsConf, checkpointLocation,
+ sleepIntervalMillis, checkpointCutOffMultiplier, aggregatorDisabledParam,
+ tableName, outputTableName, nativeTimeRangeDelay, haController);
+
+ this.metadataManagerInstance = metadataManager;
+ appAggregator = new TimelineMetricAppAggregator(metadataManager, metricsConf);
+ this.timeSliceIntervalMillis = timeSliceInterval;
+ this.serverTimeShiftAdjustment = Long.parseLong(metricsConf.get(SERVER_SIDE_TIMESIFT_ADJUSTMENT, "90000"));
+ this.interpolationEnabled = Boolean.parseBoolean(metricsConf.get(TIMELINE_METRICS_CLUSTER_AGGREGATOR_INTERPOLATION_ENABLED, "true"));
+ this.skipAggrPatternStrings = metricsConf.get(TIMELINE_METRIC_AGGREGATION_SQL_FILTERS);
+ this.timelineMetricReadHelper = new TimelineMetricReadHelper(metadataManager, true);
+ }
+
+ @Override
+ protected void aggregate(ResultSet rs, long startTime, long endTime) throws SQLException, IOException {
+ // Account for time shift due to client side buffering by shifting the
+ // timestamps with the difference between server time and series start time
+ // Also, we do not want to look at the shift time period from the end as well since we can interpolate those points
+ // that come earlier than the expected, during the next run.
+ List<Long[]> timeSlices = getTimeSlices(startTime - serverTimeShiftAdjustment, endTime - serverTimeShiftAdjustment, timeSliceIntervalMillis);
+ // Initialize app aggregates for host metrics
+ appAggregator.init();
+ Map<TimelineClusterMetric, MetricClusterAggregate> aggregateClusterMetrics =
+ aggregateMetricsFromResultSet(rs, timeSlices);
+
+ LOG.info("Saving " + aggregateClusterMetrics.size() + " metric aggregates.");
+ hBaseAccessor.saveClusterAggregateRecords(aggregateClusterMetrics);
+ appAggregator.cleanup();
+ }
+
+ @Override
+ protected Condition prepareMetricQueryCondition(long startTime, long endTime) {
+
+ List<String> metricNames = new ArrayList<>();
+ boolean metricNamesNotCondition = false;
+
+ if (!StringUtils.isEmpty(skipAggrPatternStrings)) {
+ LOG.info("Skipping aggregation for metric patterns : " + skipAggrPatternStrings);
+ metricNames.addAll(Arrays.asList(skipAggrPatternStrings.split(",")));
+ metricNamesNotCondition = true;
+ }
+
+ Condition condition = new DefaultCondition(metricNames, null, null, null, startTime - serverTimeShiftAdjustment,
+ endTime, null, null, true);
+ condition.setMetricNamesNotCondition(metricNamesNotCondition);
+ condition.setNoLimit();
+ condition.setFetchSize(resultsetFetchSize);
+ condition.setStatement(String.format(GET_METRIC_SQL,
+ METRICS_RECORD_TABLE_NAME));
+ // Retaining order of the row-key avoids client side merge sort.
+ condition.addOrderByColumn("UUID");
+ condition.addOrderByColumn("SERVER_TIME");
+ return condition;
+ }
+
+ Map<TimelineClusterMetric, MetricClusterAggregate> aggregateMetricsFromResultSet(ResultSet rs, List<Long[]> timeSlices)
+ throws SQLException, IOException {
+ Map<TimelineClusterMetric, MetricClusterAggregate> aggregateClusterMetrics =
+ new HashMap<TimelineClusterMetric, MetricClusterAggregate>();
+
+ TimelineMetric metric = null;
+ Map<String, MutableInt> hostedAppCounter = new HashMap<>();
+ if (rs.next()) {
+ metric = timelineMetricReadHelper.getTimelineMetricFromResultSet(rs);
+
+ // Call slice after all rows for a host are read
+ while (rs.next()) {
+ TimelineMetric nextMetric = timelineMetricReadHelper.getTimelineMetricFromResultSet(rs);
+ // If rows belong to same host combine them before slicing. This
+ // avoids issues across rows that belong to same hosts but get
+ // counted as coming from different ones.
+ if (metric.equalsExceptTime(nextMetric)) {
+ metric.addMetricValues(nextMetric.getMetricValues());
+ } else {
+ // Process the current metric
+ int numHosts = processAggregateClusterMetrics(aggregateClusterMetrics, metric, timeSlices);
+ if (!hostedAppCounter.containsKey(metric.getAppId())) {
+ hostedAppCounter.put(metric.getAppId(), new MutableInt(numHosts));
+ } else {
+ int currentHostCount = hostedAppCounter.get(metric.getAppId()).intValue();
+ if (currentHostCount < numHosts) {
+ hostedAppCounter.put(metric.getAppId(), new MutableInt(numHosts));
+ }
+ }
+ metric = nextMetric;
+ }
+ }
+ }
+ // Process last metric
+ if (metric != null) {
+ int numHosts = processAggregateClusterMetrics(aggregateClusterMetrics, metric, timeSlices);
+ if (!hostedAppCounter.containsKey(metric.getAppId())) {
+ hostedAppCounter.put(metric.getAppId(), new MutableInt(numHosts));
+ } else {
+ int currentHostCount = hostedAppCounter.get(metric.getAppId()).intValue();
+ if (currentHostCount < numHosts) {
+ hostedAppCounter.put(metric.getAppId(), new MutableInt(numHosts));
+ }
+ }
+ }
+
+ // Add app level aggregates to save
+ aggregateClusterMetrics.putAll(appAggregator.getAggregateClusterMetrics());
+
+ // Add liveHosts per AppId metrics.
+ long timestamp = timeSlices.get(timeSlices.size() - 1)[1];
+ processLiveAppCountMetrics(aggregateClusterMetrics, hostedAppCounter, timestamp);
+
+ return aggregateClusterMetrics;
+ }
+
+ /**
+ * Slice metric values into interval specified by :
+ * timeline.metrics.cluster.aggregator.minute.timeslice.interval
+ * Normalize value by averaging them within the interval
+ */
+ protected int processAggregateClusterMetrics(Map<TimelineClusterMetric, MetricClusterAggregate> aggregateClusterMetrics,
+ TimelineMetric metric, List<Long[]> timeSlices) {
+ // Create time slices
+ TimelineMetricMetadataKey appKey = new TimelineMetricMetadataKey(metric.getMetricName(), metric.getAppId(), metric.getInstanceId());
+ TimelineMetricMetadata metricMetadata = metadataManagerInstance.getMetadataCacheValue(appKey);
+
+ if (metricMetadata != null && !metricMetadata.isSupportsAggregates()) {
+ LOG.debug("Skipping cluster aggregation for " + metric.getMetricName());
+ return 0;
+ }
+
+ Map<TimelineClusterMetric, Double> clusterMetrics = sliceFromTimelineMetric(metric, timeSlices, interpolationEnabled);
+
+ return aggregateClusterMetricsFromSlices(clusterMetrics, aggregateClusterMetrics, metric.getHostName());
+ }
+
+ protected int aggregateClusterMetricsFromSlices(Map<TimelineClusterMetric, Double> clusterMetrics,
+ Map<TimelineClusterMetric, MetricClusterAggregate> aggregateClusterMetrics,
+ String hostname) {
+
+ int numHosts = 0;
+ if (clusterMetrics != null && !clusterMetrics.isEmpty()) {
+ for (Map.Entry<TimelineClusterMetric, Double> clusterMetricEntry : clusterMetrics.entrySet()) {
+
+ TimelineClusterMetric clusterMetric = clusterMetricEntry.getKey();
+ Double avgValue = clusterMetricEntry.getValue();
+
+ MetricClusterAggregate aggregate = aggregateClusterMetrics.get(clusterMetric);
+
+ if (aggregate == null) {
+ aggregate = new MetricClusterAggregate(avgValue, 1, null, avgValue, avgValue);
+ aggregateClusterMetrics.put(clusterMetric, aggregate);
+ } else {
+ aggregate.updateSum(avgValue);
+ aggregate.updateNumberOfHosts(1);
+ aggregate.updateMax(avgValue);
+ aggregate.updateMin(avgValue);
+ }
+
+ numHosts = aggregate.getNumberOfHosts();
+ // Update app level aggregates
+ appAggregator.processTimelineClusterMetric(clusterMetric, hostname, avgValue);
+ }
+ }
+ return numHosts;
+ }
+
+ /* Add cluster metric for number of hosts that are hosting an appId */
+ protected void processLiveAppCountMetrics(Map<TimelineClusterMetric, MetricClusterAggregate> aggregateClusterMetrics,
+ Map<String, MutableInt> appHostsCount, long timestamp) {
+
+ for (Map.Entry<String, MutableInt> appHostsEntry : appHostsCount.entrySet()) {
+ TimelineClusterMetric timelineClusterMetric = new TimelineClusterMetric(
+ liveHostsMetricName, appHostsEntry.getKey(), null, timestamp);
+
+ Integer numOfHosts = appHostsEntry.getValue().intValue();
+
+ MetricClusterAggregate metricClusterAggregate = new MetricClusterAggregate(
+ (double) numOfHosts, 1, null, (double) numOfHosts, (double) numOfHosts);
+
+ metadataManagerInstance.getUuid(timelineClusterMetric);
+
+ aggregateClusterMetrics.put(timelineClusterMetric, metricClusterAggregate);
+ }
+
+ }
+}
[12/28] ambari git commit: AMBARI-22740 : Fix integration test for
HBase in branch-3.0-ams due to UUID changes. (avijayan)
Posted by av...@apache.org.
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/query/ConnectionProvider.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/query/ConnectionProvider.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/query/ConnectionProvider.java
deleted file mode 100644
index 24239a0..0000000
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/query/ConnectionProvider.java
+++ /dev/null
@@ -1,29 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query;
-
-
-import java.sql.Connection;
-import java.sql.SQLException;
-
-/**
- *
- */
-public interface ConnectionProvider {
- public Connection getConnection() throws SQLException;
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/query/DefaultCondition.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/query/DefaultCondition.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/query/DefaultCondition.java
deleted file mode 100644
index a88f44e..0000000
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/query/DefaultCondition.java
+++ /dev/null
@@ -1,421 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query;
-
-import java.util.ArrayList;
-import java.util.LinkedHashSet;
-import java.util.List;
-import java.util.Set;
-
-import org.apache.commons.collections.CollectionUtils;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.metrics2.sink.timeline.Precision;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.PhoenixHBaseAccessor;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.discovery.TimelineMetricMetadataManager;
-
-public class DefaultCondition implements Condition {
- List<String> metricNames;
- List<String> hostnames;
- String appId;
- String instanceId;
- Long startTime;
- Long endTime;
- Precision precision;
- Integer limit;
- boolean grouped;
- boolean noLimit = false;
- Integer fetchSize;
- String statement;
- Set<String> orderByColumns = new LinkedHashSet<String>();
- boolean metricNamesNotCondition = false;
- boolean hostNamesNotCondition = false;
- boolean uuidNotCondition = false;
- List<byte[]> uuids = new ArrayList<>();
-
- private static final Log LOG = LogFactory.getLog(DefaultCondition.class);
-
- public DefaultCondition(List<String> metricNames, List<String> hostnames, String appId,
- String instanceId, Long startTime, Long endTime, Precision precision,
- Integer limit, boolean grouped) {
- this.metricNames = metricNames;
- this.hostnames = hostnames;
- this.appId = appId;
- this.instanceId = instanceId;
- this.startTime = startTime;
- this.endTime = endTime;
- this.precision = precision;
- this.limit = limit;
- this.grouped = grouped;
- }
-
- public DefaultCondition(List<byte[]> uuids, List<String> metricNames, List<String> hostnames, String appId,
- String instanceId, Long startTime, Long endTime, Precision precision,
- Integer limit, boolean grouped) {
- this.uuids = uuids;
- this.metricNames = metricNames;
- this.hostnames = hostnames;
- this.appId = appId;
- this.instanceId = instanceId;
- this.startTime = startTime;
- this.endTime = endTime;
- this.precision = precision;
- this.limit = limit;
- this.grouped = grouped;
- }
-
- public String getStatement() {
- return statement;
- }
-
- public void setStatement(String statement) {
- this.statement = statement;
- }
-
- public List<String> getMetricNames() {
- return metricNames == null || metricNames.isEmpty() ? null : metricNames;
- }
-
- public StringBuilder getConditionClause() {
- StringBuilder sb = new StringBuilder();
- boolean appendConjunction = appendUuidClause(sb);
- appendConjunction = append(sb, appendConjunction, getStartTime(), " SERVER_TIME >= ?");
- append(sb, appendConjunction, getEndTime(), " SERVER_TIME < ?");
-
- return sb;
- }
-
- protected static boolean append(StringBuilder sb,
- boolean appendConjunction,
- Object value, String str) {
- if (value != null) {
- if (appendConjunction) {
- sb.append(" AND");
- }
-
- sb.append(str);
- appendConjunction = true;
- }
- return appendConjunction;
- }
-
- public List<String> getHostnames() {
- return hostnames;
- }
-
- public Precision getPrecision() {
- return precision;
- }
-
- public void setPrecision(Precision precision) {
- this.precision = precision;
- }
-
- public String getAppId() {
- if (appId != null && !appId.isEmpty()) {
- if (!(appId.equals("HOST") || appId.equals("FLUME_HANDLER"))) {
- return appId.toLowerCase();
- } else {
- return appId;
- }
- }
- return null;
- }
-
- public String getInstanceId() {
- return instanceId == null || instanceId.isEmpty() ? null : instanceId;
- }
-
- /**
- * Convert to millis.
- */
- public Long getStartTime() {
- if (startTime == null) {
- return null;
- } else if (startTime < 9999999999l) {
- return startTime * 1000;
- } else {
- return startTime;
- }
- }
-
- public Long getEndTime() {
- if (endTime == null) {
- return null;
- }
- if (endTime < 9999999999l) {
- return endTime * 1000;
- } else {
- return endTime;
- }
- }
-
- public void setNoLimit() {
- this.noLimit = true;
- }
-
- @Override
- public boolean doUpdate() {
- return false;
- }
-
- public Integer getLimit() {
- if (noLimit) {
- return null;
- }
- return limit == null ? PhoenixHBaseAccessor.RESULTSET_LIMIT : limit;
- }
-
- public boolean isGrouped() {
- return grouped;
- }
-
- public boolean isPointInTime() {
- return getStartTime() == null && getEndTime() == null;
- }
-
- public boolean isEmpty() {
- return (metricNames == null || metricNames.isEmpty())
- && (hostnames == null || hostnames.isEmpty())
- && (appId == null || appId.isEmpty())
- && (instanceId == null || instanceId.isEmpty())
- && startTime == null
- && endTime == null;
- }
-
- public Integer getFetchSize() {
- return fetchSize;
- }
-
- public void setFetchSize(Integer fetchSize) {
- this.fetchSize = fetchSize;
- }
-
- public void addOrderByColumn(String column) {
- orderByColumns.add(column);
- }
-
- public String getOrderByClause(boolean asc) {
- String orderByStr = " ORDER BY ";
- if (!orderByColumns.isEmpty()) {
- StringBuilder sb = new StringBuilder(orderByStr);
- for (String orderByColumn : orderByColumns) {
- if (sb.length() != orderByStr.length()) {
- sb.append(", ");
- }
- sb.append(orderByColumn);
- if (!asc) {
- sb.append(" DESC");
- }
- }
- sb.append(" ");
- return sb.toString();
- }
- return null;
- }
-
- protected boolean appendUuidClause(StringBuilder sb) {
- boolean appendConjunction = false;
-
- if (CollectionUtils.isNotEmpty(uuids)) {
-
- List<byte[]> uuidsHost = new ArrayList<>();
- List<byte[]> uuidsMetric = new ArrayList<>();
- List<byte[]> uuidsFull = new ArrayList<>();
-
- if (getUuids() != null) {
- for (byte[] uuid : uuids) {
- if (uuid.length == TimelineMetricMetadataManager.TIMELINE_METRIC_UUID_LENGTH) {
- uuidsMetric.add(uuid);
- } else if (uuid.length == TimelineMetricMetadataManager.HOSTNAME_UUID_LENGTH) {
- uuidsHost.add(uuid);
- } else {
- uuidsFull.add(uuid);
- }
- }
-
- // Put a '(' first
- sb.append("(");
-
- //IN clause
- // METRIC_NAME (NOT) IN (?,?,?,?)
- if (CollectionUtils.isNotEmpty(uuidsFull)) {
- sb.append("UUID");
- if (uuidNotCondition) {
- sb.append(" NOT");
- }
- sb.append(" IN (");
- //Append ?,?,?,?
- for (int i = 0; i < uuidsFull.size(); i++) {
- sb.append("?");
- if (i < uuidsFull.size() - 1) {
- sb.append(", ");
- }
- }
- sb.append(")");
- appendConjunction = true;
- }
-
- //Put an AND if both types are present
- if (CollectionUtils.isNotEmpty(uuidsFull) &&
- CollectionUtils.isNotEmpty(uuidsMetric)) {
- sb.append(" AND ");
- }
-
- // ( for OR
- if (!metricNamesNotCondition && uuidsMetric.size() > 1 && (CollectionUtils.isNotEmpty(uuidsFull) || CollectionUtils.isNotEmpty(uuidsHost))) {
- sb.append("(");
- }
-
- //LIKE clause for clusterMetric UUIDs
- // UUID (NOT) LIKE ? OR(AND) UUID LIKE ?
- if (CollectionUtils.isNotEmpty(uuidsMetric)) {
-
- for (int i = 0; i < uuidsMetric.size(); i++) {
- sb.append("UUID");
- if (metricNamesNotCondition) {
- sb.append(" NOT");
- }
- sb.append(" LIKE ");
- sb.append("?");
-
- if (i < uuidsMetric.size() - 1) {
- if (metricNamesNotCondition) {
- sb.append(" AND ");
- } else {
- sb.append(" OR ");
- }
- // ) for OR
- } else if ((CollectionUtils.isNotEmpty(uuidsFull) || CollectionUtils.isNotEmpty(uuidsHost)) && !metricNamesNotCondition && uuidsMetric.size() > 1) {
- sb.append(")");
- }
- }
- appendConjunction = true;
- }
-
- //Put an AND if both types are present
- if ((CollectionUtils.isNotEmpty(uuidsMetric) || (CollectionUtils.isNotEmpty(uuidsFull) && CollectionUtils.isEmpty(uuidsMetric)))
- && CollectionUtils.isNotEmpty(uuidsHost)) {
- sb.append(" AND ");
- }
- // ( for OR
- if((CollectionUtils.isNotEmpty(uuidsFull) || CollectionUtils.isNotEmpty(uuidsMetric)) && !hostNamesNotCondition && uuidsHost.size() > 1){
- sb.append("(");
- }
-
- //LIKE clause for HOST UUIDs
- // UUID (NOT) LIKE ? OR(AND) UUID LIKE ?
- if (CollectionUtils.isNotEmpty(uuidsHost)) {
-
- for (int i = 0; i < uuidsHost.size(); i++) {
- sb.append("UUID");
- if (hostNamesNotCondition) {
- sb.append(" NOT");
- }
- sb.append(" LIKE ");
- sb.append("?");
-
- if (i < uuidsHost.size() - 1) {
- if (hostNamesNotCondition) {
- sb.append(" AND ");
- } else {
- sb.append(" OR ");
- }
- // ) for OR
- } else if ((CollectionUtils.isNotEmpty(uuidsFull) || CollectionUtils.isNotEmpty(uuidsMetric)) && !hostNamesNotCondition && uuidsHost.size() > 1) {
- sb.append(")");
- }
- }
- appendConjunction = true;
- }
-
- // Finish with a ')'
- if (appendConjunction) {
- sb.append(")");
- }
-
- uuids = new ArrayList<>();
- if (CollectionUtils.isNotEmpty(uuidsFull)) {
- uuids.addAll(uuidsFull);
- }
- for (byte[] uuid: uuidsMetric) {
- uuids.add(new String(uuid).concat("%").getBytes());
- }
- for (byte[] uuid: uuidsHost) {
- uuids.add("%".concat(new String(uuid)).getBytes());
- }
- }
- }
-
- return appendConjunction;
- }
-
- @Override
- public String toString() {
- return "Condition{" +
- "uuids=" + uuids +
- ", appId='" + appId + '\'' +
- ", instanceId='" + instanceId + '\'' +
- ", startTime=" + startTime +
- ", endTime=" + endTime +
- ", limit=" + limit +
- ", grouped=" + grouped +
- ", orderBy=" + orderByColumns +
- ", noLimit=" + noLimit +
- '}';
- }
-
- protected static boolean metricNamesHaveWildcard(List<String> metricNames) {
- for (String name : metricNames) {
- if (name.contains("%")) {
- return true;
- }
- }
- return false;
- }
-
- protected static boolean hostNamesHaveWildcard(List<String> hostnames) {
- if (hostnames == null)
- return false;
- for (String name : hostnames) {
- if (name.contains("%")) {
- return true;
- }
- }
- return false;
- }
-
- public void setMetricNamesNotCondition(boolean metricNamesNotCondition) {
- this.metricNamesNotCondition = metricNamesNotCondition;
- }
-
- @Override
- public void setHostnamesNotCondition(boolean hostNamesNotCondition) {
- this.hostNamesNotCondition = hostNamesNotCondition;
- }
-
- @Override
- public void setUuidNotCondition(boolean uuidNotCondition) {
- this.uuidNotCondition = uuidNotCondition;
- }
-
- @Override
- public List<byte[]> getUuids() {
- return uuids;
- }
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/query/DefaultPhoenixDataSource.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/query/DefaultPhoenixDataSource.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/query/DefaultPhoenixDataSource.java
deleted file mode 100644
index 78fad62..0000000
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/query/DefaultPhoenixDataSource.java
+++ /dev/null
@@ -1,90 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query;
-
-
-import java.io.IOException;
-import java.sql.Connection;
-import java.sql.DriverManager;
-import java.sql.SQLException;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.client.ConnectionFactory;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
-
-public class DefaultPhoenixDataSource implements PhoenixConnectionProvider {
-
- static final Log LOG = LogFactory.getLog(DefaultPhoenixDataSource.class);
- private static final String ZOOKEEPER_CLIENT_PORT = "hbase.zookeeper.property.clientPort";
- private static final String ZOOKEEPER_QUORUM = "hbase.zookeeper.quorum";
- private static final String ZNODE_PARENT = "zookeeper.znode.parent";
-
- private static final String connectionUrl = "jdbc:phoenix:%s:%s:%s";
- private final String url;
-
- private Configuration hbaseConf;
-
- public DefaultPhoenixDataSource(Configuration hbaseConf) {
- this.hbaseConf = hbaseConf;
- String zookeeperClientPort = hbaseConf.getTrimmed(ZOOKEEPER_CLIENT_PORT, "2181");
- String zookeeperQuorum = hbaseConf.getTrimmed(ZOOKEEPER_QUORUM);
- String znodeParent = hbaseConf.getTrimmed(ZNODE_PARENT, "/ams-hbase-unsecure");
- if (zookeeperQuorum == null || zookeeperQuorum.isEmpty()) {
- throw new IllegalStateException("Unable to find Zookeeper quorum to " +
- "access HBase store using Phoenix.");
- }
-
- url = String.format(connectionUrl,
- zookeeperQuorum,
- zookeeperClientPort,
- znodeParent);
- }
-
- /**
- * Get HBaseAdmin for table ops.
- * @return @HBaseAdmin
- * @throws IOException
- */
- public HBaseAdmin getHBaseAdmin() throws IOException {
- return (HBaseAdmin) ConnectionFactory.createConnection(hbaseConf).getAdmin();
- }
-
- /**
- * Get JDBC connection to HBase store. Assumption is that the hbase
- * configuration is present on the classpath and loaded by the caller into
- * the Configuration object.
- * Phoenix already caches the HConnection between the client and HBase
- * cluster.
- *
- * @return @java.sql.Connection
- */
- public Connection getConnection() throws SQLException {
-
- LOG.debug("Metric store connection url: " + url);
- try {
- return DriverManager.getConnection(url);
- } catch (SQLException e) {
- LOG.warn("Unable to connect to HBase store using Phoenix.", e);
-
- throw e;
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/query/EmptyCondition.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/query/EmptyCondition.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/query/EmptyCondition.java
deleted file mode 100644
index 5d1e244..0000000
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/query/EmptyCondition.java
+++ /dev/null
@@ -1,169 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p/>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p/>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query;
-
-import java.util.List;
-
-import org.apache.commons.lang.NotImplementedException;
-import org.apache.hadoop.metrics2.sink.timeline.Precision;
-
-/**
- * Encapsulate a Condition with pre-formatted and pre-parsed query string.
- */
-public class EmptyCondition implements Condition {
- String statement;
- boolean doUpdate = false;
- boolean metricNamesNotCondition = false;
-
- @Override
- public boolean isEmpty() {
- return false;
- }
-
- @Override
- public List<byte[]> getUuids() {
- return null;
- }
-
- @Override
- public List<String> getMetricNames() {
- return null;
- }
-
- @Override
- public boolean isPointInTime() {
- return false;
- }
-
- @Override
- public boolean isGrouped() {
- return true;
- }
-
- @Override
- public void setStatement(String statement) {
- this.statement = statement;
- }
-
- @Override
- public List<String> getHostnames() {
- return null;
- }
-
- @Override
- public Precision getPrecision() {
- return null;
- }
-
- @Override
- public void setPrecision(Precision precision) {
-
- }
-
- @Override
- public String getAppId() {
- return null;
- }
-
- @Override
- public String getInstanceId() {
- return null;
- }
-
- @Override
- public StringBuilder getConditionClause() {
- return null;
- }
-
- @Override
- public String getOrderByClause(boolean asc) {
- return null;
- }
-
- @Override
- public String getStatement() {
- return statement;
- }
-
- @Override
- public Long getStartTime() {
- return null;
- }
-
- @Override
- public Long getEndTime() {
- return null;
- }
-
- @Override
- public Integer getLimit() {
- return null;
- }
-
- @Override
- public Integer getFetchSize() {
- return null;
- }
-
- @Override
- public void setFetchSize(Integer fetchSize) {
-
- }
-
- @Override
- public void addOrderByColumn(String column) {
-
- }
-
- @Override
- public void setNoLimit() {
-
- }
-
- public void setDoUpdate(boolean doUpdate) {
- this.doUpdate = doUpdate;
- }
-
- @Override
- public boolean doUpdate() {
- return doUpdate;
- }
-
- @Override
- public String toString() {
- return "EmptyCondition{ " +
- " statement = " + this.getStatement() +
- " doUpdate = " + this.doUpdate() +
- " }";
- }
-
- @Override
- public void setMetricNamesNotCondition(boolean metricNamesNotCondition) {
- this.metricNamesNotCondition = metricNamesNotCondition;
- }
-
- @Override
- public void setHostnamesNotCondition(boolean hostNamesNotCondition) {
- throw new NotImplementedException("Not implemented");
- }
-
- @Override
- public void setUuidNotCondition(boolean uuidNotCondition) {
- throw new NotImplementedException("Not implemented");
- }
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/query/PhoenixConnectionProvider.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/query/PhoenixConnectionProvider.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/query/PhoenixConnectionProvider.java
deleted file mode 100644
index cdb3b4e..0000000
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/query/PhoenixConnectionProvider.java
+++ /dev/null
@@ -1,31 +0,0 @@
-package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query;
-
-import java.io.IOException;
-
-import org.apache.hadoop.hbase.client.Admin;
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p/>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p/>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-public interface PhoenixConnectionProvider extends ConnectionProvider {
- /**
- * Get HBaseAdmin for the Phoenix connection
- * @return
- * @throws IOException
- */
- Admin getHBaseAdmin() throws IOException;
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/query/PhoenixTransactSQL.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/query/PhoenixTransactSQL.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/query/PhoenixTransactSQL.java
deleted file mode 100644
index 457f5af..0000000
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/query/PhoenixTransactSQL.java
+++ /dev/null
@@ -1,804 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p/>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p/>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query;
-
-import java.sql.Connection;
-import java.sql.PreparedStatement;
-import java.sql.SQLException;
-import java.util.List;
-import java.util.concurrent.TimeUnit;
-import java.util.regex.Pattern;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.metrics2.sink.timeline.Precision;
-import org.apache.hadoop.metrics2.sink.timeline.PrecisionLimitExceededException;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.PhoenixHBaseAccessor;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.discovery.TimelineMetricMetadataManager;
-
-/**
- * Encapsulate all metrics related SQL queries.
- */
-public class PhoenixTransactSQL {
-
- public static final Log LOG = LogFactory.getLog(PhoenixTransactSQL.class);
-
- /**
- * Create table to store individual metric records.
- */
- public static final String CREATE_METRICS_TABLE_SQL = "CREATE TABLE IF NOT " +
- "EXISTS METRIC_RECORD (UUID BINARY(20) NOT NULL, " +
- "SERVER_TIME BIGINT NOT NULL, " +
- "METRIC_SUM DOUBLE, " +
- "METRIC_COUNT UNSIGNED_INT, " +
- "METRIC_MAX DOUBLE, " +
- "METRIC_MIN DOUBLE, " +
- "METRICS VARCHAR CONSTRAINT pk " +
- "PRIMARY KEY (UUID, SERVER_TIME ROW_TIMESTAMP)) DATA_BLOCK_ENCODING='%s', IMMUTABLE_ROWS=true, " +
- "TTL=%s, COMPRESSION='%s'";
-
- public static final String CREATE_CONTAINER_METRICS_TABLE_SQL =
- "CREATE TABLE IF NOT EXISTS CONTAINER_METRICS "
- + "(APP_ID VARCHAR, "
- + " CONTAINER_ID VARCHAR,"
- + " START_TIME TIMESTAMP,"
- + " FINISH_TIME TIMESTAMP, "
- + " DURATION BIGINT,"
- + " HOSTNAME VARCHAR,"
- + " EXIT_CODE INTEGER,"
- + " LOCALIZATION_DURATION BIGINT,"
- + " LAUNCH_DURATION BIGINT,"
- + " MEM_REQUESTED_GB DOUBLE,"
- + " MEM_REQUESTED_GB_MILLIS DOUBLE,"
- + " MEM_VIRTUAL_GB DOUBLE,"
- + " MEM_USED_GB_MIN DOUBLE,"
- + " MEM_USED_GB_MAX DOUBLE,"
- + " MEM_USED_GB_AVG DOUBLE,"
- + " MEM_USED_GB_50_PCT DOUBLE,"
- + " MEM_USED_GB_75_PCT DOUBLE,"
- + " MEM_USED_GB_90_PCT DOUBLE,"
- + " MEM_USED_GB_95_PCT DOUBLE,"
- + " MEM_USED_GB_99_PCT DOUBLE,"
- + " MEM_UNUSED_GB DOUBLE,"
- + " MEM_UNUSED_GB_MILLIS DOUBLE "
- + " CONSTRAINT pk PRIMARY KEY(APP_ID, CONTAINER_ID)) DATA_BLOCK_ENCODING='%s',"
- + " IMMUTABLE_ROWS=true, TTL=%s, COMPRESSION='%s'";
-
- public static final String CREATE_METRICS_AGGREGATE_TABLE_SQL =
- "CREATE TABLE IF NOT EXISTS %s " +
- "(UUID BINARY(20) NOT NULL, " +
- "SERVER_TIME BIGINT NOT NULL, " +
- "METRIC_SUM DOUBLE," +
- "METRIC_COUNT UNSIGNED_INT, " +
- "METRIC_MAX DOUBLE," +
- "METRIC_MIN DOUBLE CONSTRAINT pk " +
- "PRIMARY KEY (UUID, SERVER_TIME ROW_TIMESTAMP)) DATA_BLOCK_ENCODING='%s', IMMUTABLE_ROWS=true, TTL=%s," +
- " COMPRESSION='%s'";
-
- public static final String CREATE_METRICS_CLUSTER_AGGREGATE_TABLE_SQL =
- "CREATE TABLE IF NOT EXISTS %s " +
- "(UUID BINARY(16) NOT NULL, " +
- "SERVER_TIME BIGINT NOT NULL, " +
- "METRIC_SUM DOUBLE, " +
- "HOSTS_COUNT UNSIGNED_INT, " +
- "METRIC_MAX DOUBLE, " +
- "METRIC_MIN DOUBLE " +
- "CONSTRAINT pk PRIMARY KEY (UUID, SERVER_TIME ROW_TIMESTAMP)) DATA_BLOCK_ENCODING='%s', IMMUTABLE_ROWS=true, " +
- "TTL=%s, COMPRESSION='%s'";
-
- // HOSTS_COUNT vs METRIC_COUNT
- public static final String CREATE_METRICS_CLUSTER_AGGREGATE_GROUPED_TABLE_SQL =
- "CREATE TABLE IF NOT EXISTS %s " +
- "(UUID BINARY(16) NOT NULL, " +
- "SERVER_TIME BIGINT NOT NULL, " +
- "METRIC_SUM DOUBLE, " +
- "METRIC_COUNT UNSIGNED_INT, " +
- "METRIC_MAX DOUBLE, " +
- "METRIC_MIN DOUBLE " +
- "CONSTRAINT pk PRIMARY KEY (UUID, SERVER_TIME ROW_TIMESTAMP)) DATA_BLOCK_ENCODING='%s', IMMUTABLE_ROWS=true, " +
- "TTL=%s, COMPRESSION='%s'";
-
- public static final String CREATE_METRICS_METADATA_TABLE_SQL =
- "CREATE TABLE IF NOT EXISTS METRICS_METADATA " +
- "(METRIC_NAME VARCHAR, " +
- "APP_ID VARCHAR, " +
- "INSTANCE_ID VARCHAR, " +
- "UUID BINARY(16), " +
- "UNITS CHAR(20), " +
- "TYPE CHAR(20), " +
- "START_TIME UNSIGNED_LONG, " +
- "SUPPORTS_AGGREGATION BOOLEAN, " +
- "IS_WHITELISTED BOOLEAN " +
- "CONSTRAINT pk PRIMARY KEY (METRIC_NAME, APP_ID)) " +
- "DATA_BLOCK_ENCODING='%s', COMPRESSION='%s'";
-
- public static final String CREATE_HOSTED_APPS_METADATA_TABLE_SQL =
- "CREATE TABLE IF NOT EXISTS HOSTED_APPS_METADATA " +
- "(HOSTNAME VARCHAR, UUID BINARY(4), APP_IDS VARCHAR, " +
- "CONSTRAINT pk PRIMARY KEY (HOSTNAME))" +
- "DATA_BLOCK_ENCODING='%s', COMPRESSION='%s'";
-
- public static final String CREATE_INSTANCE_HOST_TABLE_SQL =
- "CREATE TABLE IF NOT EXISTS INSTANCE_HOST_METADATA " +
- "(INSTANCE_ID VARCHAR, HOSTNAME VARCHAR, " +
- "CONSTRAINT pk PRIMARY KEY (INSTANCE_ID, HOSTNAME))" +
- "DATA_BLOCK_ENCODING='%s', COMPRESSION='%s'";
-
- public static final String ALTER_METRICS_METADATA_TABLE =
- "ALTER TABLE METRICS_METADATA ADD IF NOT EXISTS IS_WHITELISTED BOOLEAN";
-
- /**
- * ALTER table to set new options
- */
- public static final String ALTER_SQL = "ALTER TABLE %s SET TTL=%s";
-
- /**
- * Insert into metric records table.
- */
- public static final String UPSERT_METRICS_SQL = "UPSERT INTO %s " +
- "(UUID, " +
- "SERVER_TIME, " +
- "METRIC_SUM, " +
- "METRIC_MAX, " +
- "METRIC_MIN, " +
- "METRIC_COUNT, " +
- "METRICS) VALUES " +
- "(?, ?, ?, ?, ?, ?, ?)";
-
- public static final String UPSERT_CONTAINER_METRICS_SQL = "UPSERT INTO %s " +
- "(APP_ID,"
- + " CONTAINER_ID,"
- + " START_TIME,"
- + " FINISH_TIME,"
- + " DURATION,"
- + " HOSTNAME,"
- + " EXIT_CODE,"
- + " LOCALIZATION_DURATION,"
- + " LAUNCH_DURATION,"
- + " MEM_REQUESTED_GB,"
- + " MEM_REQUESTED_GB_MILLIS,"
- + " MEM_VIRTUAL_GB,"
- + " MEM_USED_GB_MIN,"
- + " MEM_USED_GB_MAX,"
- + " MEM_USED_GB_AVG,"
- + " MEM_USED_GB_50_PCT,"
- + " MEM_USED_GB_75_PCT,"
- + " MEM_USED_GB_90_PCT,"
- + " MEM_USED_GB_95_PCT,"
- + " MEM_USED_GB_99_PCT,"
- + " MEM_UNUSED_GB,"
- + " MEM_UNUSED_GB_MILLIS) VALUES " +
- "(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)";
-
- public static final String UPSERT_CLUSTER_AGGREGATE_SQL = "UPSERT INTO " +
- "%s (UUID, " +
- "SERVER_TIME, " +
- "METRIC_SUM, " +
- "HOSTS_COUNT, " +
- "METRIC_MAX, " +
- "METRIC_MIN) " +
- "VALUES (?, ?, ?, ?, ?, ?)";
-
- public static final String UPSERT_CLUSTER_AGGREGATE_TIME_SQL = "UPSERT INTO" +
- " %s (UUID, SERVER_TIME, " +
- "METRIC_SUM, " +
- "METRIC_COUNT, " +
- "METRIC_MAX, " +
- "METRIC_MIN) " +
- "VALUES (?, ?, ?, ?, ?, ?)";
-
- public static final String UPSERT_AGGREGATE_RECORD_SQL = "UPSERT INTO " +
- "%s (UUID, " +
- "SERVER_TIME, " +
- "METRIC_SUM, " +
- "METRIC_MAX, " +
- "METRIC_MIN," +
- "METRIC_COUNT) " +
- "VALUES (?, ?, ?, ?, ?, ?)";
-
- public static final String UPSERT_METADATA_SQL =
- "UPSERT INTO METRICS_METADATA (METRIC_NAME, APP_ID, INSTANCE_ID, UUID, UNITS, TYPE, " +
- "START_TIME, SUPPORTS_AGGREGATION, IS_WHITELISTED) " +
- "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)";
-
- public static final String UPSERT_HOSTED_APPS_METADATA_SQL =
- "UPSERT INTO HOSTED_APPS_METADATA (HOSTNAME, UUID, APP_IDS) VALUES (?, ?, ?)";
-
- public static final String UPSERT_INSTANCE_HOST_METADATA_SQL =
- "UPSERT INTO INSTANCE_HOST_METADATA (INSTANCE_ID, HOSTNAME) VALUES (?, ?)";
-
- /**
- * Retrieve a set of rows from metrics records table.
- */
- public static final String GET_METRIC_SQL = "SELECT UUID, SERVER_TIME, " +
- "METRIC_SUM, " +
- "METRIC_MAX, " +
- "METRIC_MIN, " +
- "METRIC_COUNT, " +
- "METRICS " +
- "FROM %s";
-
- /**
- * Get latest metrics for a number of hosts
- *
- * Different queries for a number and a single hosts are used due to bug
- * in Apache Phoenix
- */
- public static final String GET_LATEST_METRIC_SQL = "SELECT %s E.UUID AS UUID, " +
- "E.SERVER_TIME AS SERVER_TIME, " +
- "E.METRIC_SUM AS METRIC_SUM, " +
- "E.METRIC_MAX AS METRIC_MAX, E.METRIC_MIN AS METRIC_MIN, " +
- "E.METRIC_COUNT AS METRIC_COUNT, E.METRICS AS METRICS " +
- "FROM %s AS E " +
- "INNER JOIN " +
- "(SELECT UUID, MAX(SERVER_TIME) AS MAX_SERVER_TIME " +
- "FROM %s " +
- "WHERE " +
- "%s " +
- "GROUP BY UUID) " +
- "AS I " +
- "ON E.UUID=I.UUID " +
- "AND E.SERVER_TIME=I.MAX_SERVER_TIME";
-
- public static final String GET_METRIC_AGGREGATE_ONLY_SQL = "SELECT UUID, " +
- "SERVER_TIME, " +
- "METRIC_SUM, " +
- "METRIC_MAX, " +
- "METRIC_MIN, " +
- "METRIC_COUNT " +
- "FROM %s";
-
- public static final String GET_CLUSTER_AGGREGATE_SQL = "SELECT " +
- "UUID, " +
- "SERVER_TIME, " +
- "METRIC_SUM, " +
- "HOSTS_COUNT, " +
- "METRIC_MAX, " +
- "METRIC_MIN " +
- "FROM %s";
-
- public static final String GET_CLUSTER_AGGREGATE_TIME_SQL = "SELECT " +
- "UUID, " +
- "SERVER_TIME, " +
- "METRIC_SUM, " +
- "METRIC_COUNT, " +
- "METRIC_MAX, " +
- "METRIC_MIN " +
- "FROM %s";
-
- public static final String TOP_N_INNER_SQL = "SELECT UUID " +
- "FROM %s WHERE %s GROUP BY UUID ORDER BY %s LIMIT %s";
-
- public static final String GET_METRIC_METADATA_SQL = "SELECT " +
- "METRIC_NAME, APP_ID, INSTANCE_ID, UUID, UNITS, TYPE, START_TIME, " +
- "SUPPORTS_AGGREGATION, IS_WHITELISTED FROM METRICS_METADATA";
-
- public static final String GET_HOSTED_APPS_METADATA_SQL = "SELECT " +
- "HOSTNAME, UUID, APP_IDS FROM HOSTED_APPS_METADATA";
-
- public static final String GET_INSTANCE_HOST_METADATA_SQL = "SELECT " +
- "INSTANCE_ID, HOSTNAME FROM INSTANCE_HOST_METADATA";
-
- /**
- * Aggregate host metrics using a GROUP BY clause to take advantage of
- * N - way parallel scan where N = number of regions.
- */
- public static final String GET_AGGREGATED_HOST_METRIC_GROUPBY_SQL = "UPSERT " +
- "INTO %s (UUID, SERVER_TIME, METRIC_SUM, METRIC_COUNT, METRIC_MAX, METRIC_MIN) " +
- "SELECT UUID, %s AS SERVER_TIME, " +
- "SUM(METRIC_SUM), SUM(METRIC_COUNT), MAX(METRIC_MAX), MIN(METRIC_MIN) " +
- "FROM %s WHERE%s SERVER_TIME > %s AND SERVER_TIME <= %s " +
- "GROUP BY UUID";
-
- /**
- * Downsample host metrics.
- */
- public static final String DOWNSAMPLE_HOST_METRIC_SQL_UPSERT_PREFIX = "UPSERT INTO %s (UUID, SERVER_TIME, " +
- "METRIC_SUM, METRIC_COUNT, METRIC_MAX, METRIC_MIN) ";
-
- public static final String TOPN_DOWNSAMPLER_HOST_METRIC_SELECT_SQL = "SELECT UUID, " +
- "%s AS SERVER_TIME, %s, 1, %s, %s FROM %s WHERE UUID IN %s AND SERVER_TIME > %s AND SERVER_TIME <= %s " +
- "GROUP BY UUID ORDER BY %s DESC LIMIT %s";
-
- /**
- * Aggregate app metrics using a GROUP BY clause to take advantage of
- * N - way parallel scan where N = number of regions.
- */
- public static final String GET_AGGREGATED_APP_METRIC_GROUPBY_SQL = "UPSERT " +
- "INTO %s (UUID, SERVER_TIME, METRIC_SUM, METRIC_COUNT, METRIC_MAX, METRIC_MIN) SELECT UUID, %s AS SERVER_TIME, " +
- "ROUND(AVG(METRIC_SUM),2), ROUND(AVG(%s)), MAX(METRIC_MAX), MIN(METRIC_MIN) FROM %s WHERE%s SERVER_TIME > %s AND " +
- "SERVER_TIME <= %s GROUP BY UUID";
-
- /**
- * Downsample cluster metrics.
- */
- public static final String DOWNSAMPLE_CLUSTER_METRIC_SQL_UPSERT_PREFIX = "UPSERT INTO %s (UUID, SERVER_TIME, " +
- "METRIC_SUM, METRIC_COUNT, METRIC_MAX, METRIC_MIN) ";
-
- public static final String TOPN_DOWNSAMPLER_CLUSTER_METRIC_SELECT_SQL = "SELECT UUID, " +
- "%s AS SERVER_TIME, %s, 1, %s, %s FROM %s WHERE UUID IN %s AND SERVER_TIME > %s AND SERVER_TIME <= %s " +
- "GROUP BY UUID ORDER BY %s DESC LIMIT %s";
-
- public static final String METRICS_RECORD_TABLE_NAME = "METRIC_RECORD";
-
- public static final String CONTAINER_METRICS_TABLE_NAME = "CONTAINER_METRICS";
-
- public static final String METRICS_AGGREGATE_MINUTE_TABLE_NAME =
- "METRIC_RECORD_MINUTE";
- public static final String METRICS_AGGREGATE_HOURLY_TABLE_NAME =
- "METRIC_RECORD_HOURLY";
- public static final String METRICS_AGGREGATE_DAILY_TABLE_NAME =
- "METRIC_RECORD_DAILY";
- public static final String METRICS_CLUSTER_AGGREGATE_TABLE_NAME =
- "METRIC_AGGREGATE";
- public static final String METRICS_CLUSTER_AGGREGATE_MINUTE_TABLE_NAME =
- "METRIC_AGGREGATE_MINUTE";
- public static final String METRICS_CLUSTER_AGGREGATE_HOURLY_TABLE_NAME =
- "METRIC_AGGREGATE_HOURLY";
- public static final String METRICS_CLUSTER_AGGREGATE_DAILY_TABLE_NAME =
- "METRIC_AGGREGATE_DAILY";
-
- public static final Pattern PHOENIX_TABLES_REGEX_PATTERN = Pattern.compile("METRIC_");
-
- public static final String[] PHOENIX_TABLES = {
- METRICS_RECORD_TABLE_NAME,
- METRICS_AGGREGATE_MINUTE_TABLE_NAME,
- METRICS_AGGREGATE_HOURLY_TABLE_NAME,
- METRICS_AGGREGATE_DAILY_TABLE_NAME,
- METRICS_CLUSTER_AGGREGATE_TABLE_NAME,
- METRICS_CLUSTER_AGGREGATE_MINUTE_TABLE_NAME,
- METRICS_CLUSTER_AGGREGATE_HOURLY_TABLE_NAME,
- METRICS_CLUSTER_AGGREGATE_DAILY_TABLE_NAME
- };
-
- public static final String DEFAULT_TABLE_COMPRESSION = "SNAPPY";
- public static final String DEFAULT_ENCODING = "FAST_DIFF";
- public static final long HOUR = 3600000; // 1 hour
- public static final long DAY = 86400000; // 1 day
- private static boolean sortMergeJoinEnabled = false;
-
- /**
- * Filter to optimize HBase scan by using file timestamps. This prevents
- * a full table scan of metric records.
- *
- * @return Phoenix Hint String
- */
- public static String getNaiveTimeRangeHint(Long startTime, Long delta) {
- return String.format("/*+ NATIVE_TIME_RANGE(%s) */", (startTime - delta));
- }
-
- /**
- * Falling back to sort merge join algorithm if default queries fail.
- *
- * @return Phoenix Hint String
- */
- public static String getLatestMetricsHints() {
- if (sortMergeJoinEnabled) {
- return "/*+ USE_SORT_MERGE_JOIN NO_CACHE */";
- }
- return "";
- }
-
- public static void setSortMergeJoinEnabled(boolean sortMergeJoinEnabled) {
- PhoenixTransactSQL.sortMergeJoinEnabled = sortMergeJoinEnabled;
- }
-
- public static PreparedStatement prepareGetMetricsSqlStmt(Connection connection,
- Condition condition) throws SQLException {
-
- validateConditionIsNotEmpty(condition);
- validateRowCountLimit(condition);
-
- String stmtStr;
- if (condition.getStatement() != null) {
- stmtStr = condition.getStatement();
- } else {
- String metricsTable;
- String query;
- if (condition.getPrecision() == null) {
- long endTime = condition.getEndTime() == null ? System.currentTimeMillis() : condition.getEndTime();
- long startTime = condition.getStartTime() == null ? 0 : condition.getStartTime();
- Precision precision = Precision.getPrecision(startTime, endTime);
- condition.setPrecision(precision);
- }
- switch (condition.getPrecision()) {
- case DAYS:
- metricsTable = METRICS_AGGREGATE_DAILY_TABLE_NAME;
- query = GET_METRIC_AGGREGATE_ONLY_SQL;
- break;
- case HOURS:
- metricsTable = METRICS_AGGREGATE_HOURLY_TABLE_NAME;
- query = GET_METRIC_AGGREGATE_ONLY_SQL;
- break;
- case MINUTES:
- metricsTable = METRICS_AGGREGATE_MINUTE_TABLE_NAME;
- query = GET_METRIC_AGGREGATE_ONLY_SQL;
- break;
- default:
- metricsTable = METRICS_RECORD_TABLE_NAME;
- query = GET_METRIC_SQL;
- }
-
- stmtStr = String.format(query, metricsTable);
- }
-
- StringBuilder sb = new StringBuilder(stmtStr);
-
- if (!(condition instanceof EmptyCondition)) {
- sb.append(" WHERE ");
- sb.append(condition.getConditionClause());
- String orderByClause = condition.getOrderByClause(true);
- if (orderByClause != null) {
- sb.append(orderByClause);
- } else {
- sb.append(" ORDER BY UUID, SERVER_TIME ");
- }
- }
-
- if (condition.getLimit() != null) {
- sb.append(" LIMIT ").append(condition.getLimit());
- }
-
- if (LOG.isDebugEnabled()) {
- LOG.debug("SQL: " + sb.toString() + ", condition: " + condition);
- }
-
- PreparedStatement stmt = null;
- try {
- stmt = connection.prepareStatement(sb.toString());
- int pos = 1;
- pos = addUuids(condition, pos, stmt);
-
- if (condition instanceof TopNCondition) {
- pos = addStartTime(condition, pos, stmt);
- pos = addEndTime(condition, pos, stmt);
- }
-
- pos = addStartTime(condition, pos, stmt);
- addEndTime(condition, pos, stmt);
-
- if (condition.getFetchSize() != null) {
- stmt.setFetchSize(condition.getFetchSize());
- }
- } catch (SQLException e) {
- if (stmt != null) {
- stmt.close();
- }
- throw e;
- }
-
- if (condition instanceof TopNCondition) {
- LOG.info(sb.toString());
- }
- return stmt;
- }
-
- private static void validateConditionIsNotEmpty(Condition condition) {
- if (condition.isEmpty()) {
- throw new IllegalArgumentException("Condition is empty.");
- }
- }
-
- private static void validateRowCountLimit(Condition condition) {
- if (condition.getMetricNames() == null
- || condition.getMetricNames().isEmpty()) {
- //aggregator can use empty metrics query
- return;
- }
-
- long range = condition.getEndTime() - condition.getStartTime();
- long rowsPerMetric;
-
- //Get Precision (passed in or computed) and estimate values returned based on that.
- Precision precision = condition.getPrecision();
- if (precision == null) {
- precision = Precision.getPrecision(condition.getStartTime(), condition.getEndTime());
- }
-
- switch (precision) {
- case DAYS:
- rowsPerMetric = TimeUnit.MILLISECONDS.toDays(range);
- break;
- case HOURS:
- rowsPerMetric = TimeUnit.MILLISECONDS.toHours(range);
- break;
- case MINUTES:
- rowsPerMetric = TimeUnit.MILLISECONDS.toMinutes(range)/5; //5 minute data in METRIC_AGGREGATE_MINUTE table.
- break;
- default:
- rowsPerMetric = TimeUnit.MILLISECONDS.toSeconds(range)/10; //10 second data in METRIC_AGGREGATE table
- }
-
- List<String> hostNames = condition.getHostnames();
- int numHosts = (hostNames == null || hostNames.isEmpty()) ? 1 : condition.getHostnames().size();
-
- long totalRowsRequested = rowsPerMetric * condition.getMetricNames().size() * numHosts;
-
- if (totalRowsRequested > PhoenixHBaseAccessor.RESULTSET_LIMIT) {
- throw new PrecisionLimitExceededException("Requested " + condition.getMetricNames().size() + " metrics for "
- + numHosts + " hosts in " + precision + " precision for the time range of " + range/1000
- + " seconds. Estimated resultset size of " + totalRowsRequested + " is greater than the limit of "
- + PhoenixHBaseAccessor.RESULTSET_LIMIT + ". Request lower precision or fewer number of metrics or hosts." +
- " Alternatively, increase the limit value through ams-site:timeline.metrics.service.default.result.limit config");
- }
- }
-
- public static PreparedStatement prepareGetLatestMetricSqlStmt(
- Connection connection, Condition condition) throws SQLException {
-
- validateConditionIsNotEmpty(condition);
-
- if (condition.getMetricNames() == null
- || condition.getMetricNames().isEmpty()) {
- throw new IllegalArgumentException("Point in time query without "
- + "metric names not supported ");
- }
-
- String stmtStr;
- if (condition.getStatement() != null) {
- stmtStr = condition.getStatement();
- } else {
- stmtStr = String.format(GET_LATEST_METRIC_SQL,
- getLatestMetricsHints(),
- METRICS_RECORD_TABLE_NAME,
- METRICS_RECORD_TABLE_NAME,
- condition.getConditionClause());
- }
-
- if (LOG.isDebugEnabled()) {
- LOG.debug("SQL: " + stmtStr + ", condition: " + condition);
- }
- PreparedStatement stmt = null;
- try {
- stmt = connection.prepareStatement(stmtStr);
- setQueryParameters(stmt, condition);
- } catch (SQLException e) {
- if (stmt != null) {
- stmt.close();
- }
- throw e;
- }
-
- return stmt;
- }
-
- private static PreparedStatement setQueryParameters(PreparedStatement stmt,
- Condition condition) throws SQLException {
- int pos = 1;
- //For GET_LATEST_METRIC_SQL_SINGLE_HOST parameters should be set 2 times
- do {
- if (condition.getUuids() != null) {
- for (byte[] uuid : condition.getUuids()) {
- stmt.setBytes(pos++, uuid);
- }
- }
- if (condition.getFetchSize() != null) {
- stmt.setFetchSize(condition.getFetchSize());
- pos++;
- }
- } while (pos < stmt.getParameterMetaData().getParameterCount());
-
- return stmt;
- }
-
- public static PreparedStatement prepareGetAggregateSqlStmt(
- Connection connection, Condition condition) throws SQLException {
-
- validateConditionIsNotEmpty(condition);
- validateRowCountLimit(condition);
-
- String metricsAggregateTable;
- String queryStmt;
- if (condition.getPrecision() == null) {
- long endTime = condition.getEndTime() == null ? System.currentTimeMillis() : condition.getEndTime();
- long startTime = condition.getStartTime() == null ? 0 : condition.getStartTime();
- condition.setPrecision(Precision.getPrecision(startTime, endTime));
- }
- switch (condition.getPrecision()) {
- case DAYS:
- metricsAggregateTable = METRICS_CLUSTER_AGGREGATE_DAILY_TABLE_NAME;
- queryStmt = GET_CLUSTER_AGGREGATE_TIME_SQL;
- break;
- case HOURS:
- metricsAggregateTable = METRICS_CLUSTER_AGGREGATE_HOURLY_TABLE_NAME;
- queryStmt = GET_CLUSTER_AGGREGATE_TIME_SQL;
- break;
- case MINUTES:
- metricsAggregateTable = METRICS_CLUSTER_AGGREGATE_MINUTE_TABLE_NAME;
- queryStmt = GET_CLUSTER_AGGREGATE_TIME_SQL;
- break;
- default:
- metricsAggregateTable = METRICS_CLUSTER_AGGREGATE_TABLE_NAME;
- queryStmt = GET_CLUSTER_AGGREGATE_SQL;
- }
-
- queryStmt = String.format(queryStmt, metricsAggregateTable);
-
- StringBuilder sb = new StringBuilder(queryStmt);
- sb.append(" WHERE ");
- sb.append(condition.getConditionClause());
- sb.append(" ORDER BY UUID, SERVER_TIME");
- if (condition.getLimit() != null) {
- sb.append(" LIMIT ").append(condition.getLimit());
- }
-
- String query = sb.toString();
-
- if (LOG.isDebugEnabled()) {
- LOG.debug("SQL => " + query + ", condition => " + condition);
- }
- PreparedStatement stmt = null;
- try {
- stmt = connection.prepareStatement(query);
- int pos = 1;
-
- pos = addUuids(condition, pos, stmt);
-
- if (condition instanceof TopNCondition) {
- pos = addStartTime(condition, pos, stmt);
- pos = addEndTime(condition, pos, stmt);
- }
-
- // TODO: Upper case all strings on POST
- pos = addStartTime(condition, pos, stmt);
- addEndTime(condition, pos, stmt);
- } catch (SQLException e) {
- if (stmt != null) {
- stmt.close();
- }
- throw e;
- }
-
- if (condition instanceof TopNCondition) {
- LOG.info(sb.toString());
- }
- return stmt;
- }
-
- public static PreparedStatement prepareGetLatestAggregateMetricSqlStmt(
- Connection connection, SplitByMetricNamesCondition condition) throws SQLException {
-
- validateConditionIsNotEmpty(condition);
-
- String stmtStr;
- if (condition.getStatement() != null) {
- stmtStr = condition.getStatement();
- } else {
- stmtStr = String.format(GET_CLUSTER_AGGREGATE_SQL,
- METRICS_CLUSTER_AGGREGATE_TABLE_NAME);
- }
-
- StringBuilder sb = new StringBuilder(stmtStr);
- sb.append(" WHERE ");
- sb.append(condition.getConditionClause());
- String orderByClause = condition.getOrderByClause(false);
- if (orderByClause != null) {
- sb.append(orderByClause);
- } else {
- sb.append(" ORDER BY UUID DESC, SERVER_TIME DESC ");
- }
-
- sb.append(" LIMIT ").append(condition.getMetricNames().size());
-
- String query = sb.toString();
- if (LOG.isDebugEnabled()) {
- LOG.debug("SQL: " + query + ", condition: " + condition);
- }
-
- PreparedStatement stmt = null;
- try {
- stmt = connection.prepareStatement(query);
- int pos = 1;
- if (condition.getMetricNames() != null) {
- for (; pos <= condition.getMetricNames().size(); pos++) {
- stmt.setBytes(pos, condition.getCurrentUuid());
- }
- }
- } catch (SQLException e) {
- if (stmt != null) {
-
- }
- throw e;
- }
-
- return stmt;
- }
-
- public static String getTargetTableUsingPrecision(Precision precision, boolean withHosts) {
-
- String inputTable = null;
- if (precision != null) {
- if (withHosts) {
- switch (precision) {
- case DAYS:
- inputTable = PhoenixTransactSQL.METRICS_AGGREGATE_DAILY_TABLE_NAME;
- break;
- case HOURS:
- inputTable = PhoenixTransactSQL.METRICS_AGGREGATE_HOURLY_TABLE_NAME;
- break;
- case MINUTES:
- inputTable = PhoenixTransactSQL.METRICS_AGGREGATE_MINUTE_TABLE_NAME;
- break;
- default:
- inputTable = PhoenixTransactSQL.METRICS_RECORD_TABLE_NAME;
- }
- } else {
- switch (precision) {
- case DAYS:
- inputTable = PhoenixTransactSQL.METRICS_CLUSTER_AGGREGATE_DAILY_TABLE_NAME;
- break;
- case HOURS:
- inputTable = PhoenixTransactSQL.METRICS_CLUSTER_AGGREGATE_HOURLY_TABLE_NAME;
- break;
- case MINUTES:
- inputTable = PhoenixTransactSQL.METRICS_CLUSTER_AGGREGATE_MINUTE_TABLE_NAME;
- break;
- default:
- inputTable = PhoenixTransactSQL.METRICS_CLUSTER_AGGREGATE_TABLE_NAME;
- }
- }
- } else {
- if (withHosts) {
- inputTable = PhoenixTransactSQL.METRICS_RECORD_TABLE_NAME;
- } else {
- inputTable = PhoenixTransactSQL.METRICS_CLUSTER_AGGREGATE_TABLE_NAME;
- }
- }
- return inputTable;
- }
-
- private static int addUuids(Condition condition, int pos, PreparedStatement stmt) throws SQLException {
- if (condition.getUuids() != null) {
- for (int pos2 = 1 ; pos2 <= condition.getUuids().size(); pos2++,pos++) {
- byte[] uuid = condition.getUuids().get(pos2 - 1);
- if (LOG.isDebugEnabled()) {
- LOG.debug("Setting pos: " + pos + ", value = " + new String(uuid));
- }
-
- if (uuid.length != TimelineMetricMetadataManager.HOSTNAME_UUID_LENGTH + TimelineMetricMetadataManager.TIMELINE_METRIC_UUID_LENGTH) {
- stmt.setString(pos, new String(uuid));
- } else {
- stmt.setBytes(pos, uuid);
- }
- }
- }
- return pos;
- }
-
- private static int addStartTime(Condition condition, int pos, PreparedStatement stmt) throws SQLException {
- if (condition.getStartTime() != null) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Setting pos: " + pos + ", value: " + condition.getStartTime());
- }
- stmt.setLong(pos++, condition.getStartTime());
- }
- return pos;
- }
-
- private static int addEndTime(Condition condition, int pos, PreparedStatement stmt) throws SQLException {
-
- if (condition.getEndTime() != null) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Setting pos: " + pos + ", value: " + condition.getEndTime());
- }
- stmt.setLong(pos++, condition.getEndTime());
- }
- return pos;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/query/SplitByMetricNamesCondition.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/query/SplitByMetricNamesCondition.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/query/SplitByMetricNamesCondition.java
deleted file mode 100644
index 6eadcea..0000000
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/query/SplitByMetricNamesCondition.java
+++ /dev/null
@@ -1,189 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query;
-
-import java.util.Collections;
-import java.util.List;
-
-import org.apache.hadoop.metrics2.sink.timeline.Precision;
-// TODO get rid of this class
-public class SplitByMetricNamesCondition implements Condition {
- private final Condition adaptee;
- private byte[] currentUuid;
- private boolean metricNamesNotCondition = false;
-
- public SplitByMetricNamesCondition(Condition condition){
- this.adaptee = condition;
- }
-
- @Override
- public boolean isEmpty() {
- return adaptee.isEmpty();
- }
-
- @Override
- public List<byte[]> getUuids() {
- return adaptee.getUuids();
- }
-
- @Override
- public List<String> getMetricNames() {
- return Collections.singletonList(new String(currentUuid));
- }
-
- @Override
- public boolean isPointInTime() {
- return adaptee.isPointInTime();
- }
-
- @Override
- public boolean isGrouped() {
- return adaptee.isGrouped();
- }
-
- @Override
- public void setStatement(String statement) {
- adaptee.setStatement(statement);
- }
-
- @Override
- public List<String> getHostnames() {
- return adaptee.getHostnames();
- }
-
- @Override
- public Precision getPrecision() {
- return adaptee.getPrecision();
- }
-
- @Override
- public void setPrecision(Precision precision) {
- adaptee.setPrecision(precision);
- }
-
- @Override
- public String getAppId() {
- return adaptee.getAppId();
- }
-
- @Override
- public String getInstanceId() {
- return adaptee.getInstanceId();
- }
-
- @Override
- public StringBuilder getConditionClause() {
- StringBuilder sb = new StringBuilder();
- boolean appendConjunction = false;
-
- if (getMetricNames() != null) {
- for (String name : getMetricNames()) {
- if (sb.length() > 1) {
- sb.append(" OR ");
- }
- sb.append("UUID = ?");
- }
-
- appendConjunction = true;
- }
-
- appendConjunction = DefaultCondition.append(sb, appendConjunction,
- getStartTime(), " SERVER_TIME >= ?");
- DefaultCondition.append(sb, appendConjunction, getEndTime(),
- " SERVER_TIME < ?");
-
- return sb;
- }
-
- @Override
- public String getOrderByClause(boolean asc) {
- return adaptee.getOrderByClause(asc);
- }
-
- @Override
- public String getStatement() {
- return adaptee.getStatement();
- }
-
- @Override
- public Long getStartTime() {
- return adaptee.getStartTime();
- }
-
- @Override
- public Long getEndTime() {
- return adaptee.getEndTime();
- }
-
- @Override
- public Integer getLimit() {
- return adaptee.getLimit();
- }
-
- @Override
- public Integer getFetchSize() {
- return adaptee.getFetchSize();
- }
-
- @Override
- public void setFetchSize(Integer fetchSize) {
- adaptee.setFetchSize(fetchSize);
- }
-
- @Override
- public void addOrderByColumn(String column) {
- adaptee.addOrderByColumn(column);
- }
-
- @Override
- public void setNoLimit() {
- adaptee.setNoLimit();
- }
-
- @Override
- public boolean doUpdate() {
- return false;
- }
-
- public List<String> getOriginalMetricNames() {
- return adaptee.getMetricNames();
- }
-
- public void setCurrentUuid(byte[] uuid) {
- this.currentUuid = uuid;
- }
-
- public byte[] getCurrentUuid() {
- return currentUuid;
- }
-
- @Override
- public void setMetricNamesNotCondition(boolean metricNamesNotCondition) {
- this.metricNamesNotCondition = metricNamesNotCondition;
- }
-
- @Override
- public void setHostnamesNotCondition(boolean hostNamesNotCondition) {
-
- }
-
- @Override
- public void setUuidNotCondition(boolean uuidNotCondition) {
-
- }
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/query/TopNCondition.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/query/TopNCondition.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/query/TopNCondition.java
deleted file mode 100644
index 4a5491f..0000000
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/query/TopNCondition.java
+++ /dev/null
@@ -1,162 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query;
-
-import java.util.List;
-
-import org.apache.commons.collections.CollectionUtils;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.metrics2.sink.timeline.Precision;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.aggregators.Function;
-
-public class TopNCondition extends DefaultCondition{
-
- private Integer topN;
- private boolean isBottomN;
- private Function topNFunction;
- private static final Log LOG = LogFactory.getLog(TopNCondition.class);
-
- public TopNCondition(List<byte[]> uuids, List<String> metricNames, List<String> hostnames, String appId,
- String instanceId, Long startTime, Long endTime, Precision precision,
- Integer limit, boolean grouped, Integer topN, Function topNFunction,
- boolean isBottomN) {
- super(uuids, metricNames, hostnames, appId, instanceId, startTime, endTime, precision, limit, grouped);
- this.topN = topN;
- this.isBottomN = isBottomN;
- this.topNFunction = topNFunction;
- }
-
- @Override
- public StringBuilder getConditionClause() {
-
-
- if (!(isTopNHostCondition(metricNames, hostnames) || isTopNMetricCondition(metricNames, hostnames))) {
- LOG.error("Unsupported TopN Operation requested. Query can have either multiple hosts or multiple metric names " +
- "but not both.");
- return null;
- }
-
- StringBuilder sb = new StringBuilder();
- sb.append(" UUID IN (");
- sb.append(getTopNInnerQuery());
- sb.append(")");
-
- boolean appendConjunction = true;
- appendConjunction = append(sb, appendConjunction, getStartTime(), " SERVER_TIME >= ?");
- append(sb, appendConjunction, getEndTime(), " SERVER_TIME < ?");
-
- return sb;
- }
-
- public String getTopNInnerQuery() {
- return String.format(PhoenixTransactSQL.TOP_N_INNER_SQL,
- PhoenixTransactSQL.getTargetTableUsingPrecision(precision, CollectionUtils.isNotEmpty(hostnames)),
- super.getConditionClause().toString(), getTopNOrderByClause(), topN);
- }
-
- private String getTopNOrderByClause() {
-
- String orderByClause = getColumnSelect(this.topNFunction);
- orderByClause += (isBottomN ? " ASC" : " DESC");
- return orderByClause;
- }
-
- public static String getColumnSelect(Function topNFunction) {
- String columnSelect = null;
- if (topNFunction != null) {
- switch (topNFunction.getReadFunction()) {
- case AVG:
- columnSelect = "ROUND(AVG(METRIC_SUM),2)";
- break;
- case SUM:
- columnSelect = "SUM(METRIC_SUM)";
- break;
- default:
- columnSelect = "MAX(METRIC_MAX)";
- break;
- }
- }
- if (columnSelect == null) {
- columnSelect = "MAX(METRIC_MAX)";
- }
- return columnSelect;
- }
-
- public boolean isTopNHostCondition() {
- return isTopNHostCondition(metricNames, hostnames);
- }
-
- public boolean isTopNMetricCondition() {
- return isTopNMetricCondition(metricNames, hostnames);
- }
-
- /**
- * Check if this is a case of Top N hosts condition
- * @param metricNames A list of Strings.
- * @param hostnames A list of Strings.
- * @return True if it is a Case of Top N Hosts (1 Metric and H hosts).
- */
- public static boolean isTopNHostCondition(List<String> metricNames, List<String> hostnames) {
- // Case 1 : 1 Metric, H hosts
- // Select Top N or Bottom N host series based on 1 metric (max/avg/sum)
- // Hostnames cannot be empty
- // Only 1 metric allowed, without wildcards
- return (CollectionUtils.isNotEmpty(hostnames) && metricNames.size() == 1 && !metricNamesHaveWildcard(metricNames));
-
- }
-
- /**
- * Check if this is a case of Top N metrics condition
- * @param metricNames A list of Strings.
- * @param hostnames A list of Strings.
- * @return True if it is a Case of Top N Metrics (M Metric and 1 or 0 host).
- */
- public static boolean isTopNMetricCondition(List<String> metricNames, List<String> hostnames) {
- // Case 2 : M Metric names or Regex, 1 or No host
- // Select Top N or Bottom N metric series based on metric values(max/avg/sum)
- // MetricNames cannot be empty
- // No host (aggregate) or 1 host allowed, without wildcards
- return (CollectionUtils.isNotEmpty(metricNames) && (hostnames == null || hostnames.size() <= 1) &&
- !hostNamesHaveWildcard(hostnames));
- }
-
- public Integer getTopN() {
- return topN;
- }
-
- public void setTopN(Integer topN) {
- this.topN = topN;
- }
-
- public boolean isBottomN() {
- return isBottomN;
- }
-
- public void setIsBottomN(boolean isBottomN) {
- this.isBottomN = isBottomN;
- }
-
- public Function getTopNFunction() {
- return topNFunction;
- }
-
- public void setTopNFunction(Function topNFunction) {
- this.topNFunction = topNFunction;
- }
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/sink/DefaultFSSinkProvider.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/sink/DefaultFSSinkProvider.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/sink/DefaultFSSinkProvider.java
deleted file mode 100644
index 6ec6cf9..0000000
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/sink/DefaultFSSinkProvider.java
+++ /dev/null
@@ -1,153 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.sink;
-
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.TIMELINE_METRICS_CACHE_COMMIT_INTERVAL;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.Collection;
-import java.util.Date;
-
-import org.apache.commons.io.FileUtils;
-import org.apache.commons.lang.StringUtils;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.metrics2.sink.timeline.TimelineMetric;
-import org.apache.hadoop.metrics2.sink.timeline.TimelineMetrics;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.source.InternalSourceProvider;
-
-public class DefaultFSSinkProvider implements ExternalSinkProvider {
- private static final Log LOG = LogFactory.getLog(DefaultFSSinkProvider.class);
- TimelineMetricConfiguration conf = TimelineMetricConfiguration.getInstance();
- private final DefaultExternalMetricsSink sink = new DefaultExternalMetricsSink();
- private long FIXED_FILE_SIZE;
- private final String SINK_FILE_NAME = "external-metrics-sink.dat";
- private final String SEPARATOR = ", ";
- private final String LINE_SEP = System.lineSeparator();
- private final String HEADERS = "METRIC, APP_ID, INSTANCE_ID, HOSTNAME, START_TIME, DATA";
-
- public DefaultFSSinkProvider() {
- try {
- FIXED_FILE_SIZE = conf.getMetricsConf().getLong("timeline.metrics.service.external.fs.sink.filesize", FileUtils.ONE_MB * 100);
- } catch (Exception ignored) {
- FIXED_FILE_SIZE = FileUtils.ONE_MB * 100;
- }
- }
-
- @Override
- public ExternalMetricsSink getExternalMetricsSink(InternalSourceProvider.SOURCE_NAME sourceName) {
- return sink;
- }
-
- class DefaultExternalMetricsSink implements ExternalMetricsSink {
-
- @Override
- public int getSinkTimeOutSeconds() {
- return 10;
- }
-
- @Override
- public int getFlushSeconds() {
- try {
- return conf.getMetricsConf().getInt(TIMELINE_METRICS_CACHE_COMMIT_INTERVAL, 3);
- } catch (Exception e) {
- LOG.warn("Cannot read cache commit interval.");
- }
- return 3;
- }
-
- private boolean createFile(File f) {
- boolean created = false;
- if (!f.exists()) {
- try {
- created = f.createNewFile();
- FileUtils.writeStringToFile(f, HEADERS);
- } catch (IOException e) {
- LOG.error("Cannot create " + SINK_FILE_NAME + " at " + f.getPath());
- return false;
- }
- }
-
- return created;
- }
-
- private boolean shouldReCreate(File f) {
- if (!f.exists()) {
- return true;
- }
- if (FileUtils.sizeOf(f) > FIXED_FILE_SIZE) {
- return true;
- }
- return false;
- }
-
- @Override
- public void sinkMetricData(Collection<TimelineMetrics> metrics) {
- String dirPath = TimelineMetricConfiguration.getInstance().getDefaultMetricsSinkDir();
- File dir = new File(dirPath);
- if (!dir.exists()) {
- LOG.error("Cannot sink data to file system, incorrect dir path " + dirPath);
- return;
- }
-
- File f = FileUtils.getFile(dirPath, SINK_FILE_NAME);
- if (shouldReCreate(f)) {
- if (!f.delete()) {
- LOG.warn("Unable to delete external sink file.");
- return;
- }
- createFile(f);
- }
-
- if (metrics != null) {
- for (TimelineMetrics timelineMetrics : metrics) {
- for (TimelineMetric metric : timelineMetrics.getMetrics()) {
- StringBuilder sb = new StringBuilder();
- sb.append(metric.getMetricName());
- sb.append(SEPARATOR);
- sb.append(metric.getAppId());
- sb.append(SEPARATOR);
- if (StringUtils.isEmpty(metric.getInstanceId())) {
- sb.append(SEPARATOR);
- } else {
- sb.append(metric.getInstanceId());
- sb.append(SEPARATOR);
- }
- if (StringUtils.isEmpty(metric.getHostName())) {
- sb.append(SEPARATOR);
- } else {
- sb.append(metric.getHostName());
- sb.append(SEPARATOR);
- }
- sb.append(new Date(metric.getStartTime()));
- sb.append(SEPARATOR);
- sb.append(metric.getMetricValues().toString());
- sb.append(LINE_SEP);
- try {
- FileUtils.writeStringToFile(f, sb.toString());
- } catch (IOException e) {
- LOG.warn("Unable to sink data to file " + f.getPath());
- }
- }
- }
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/sink/ExternalMetricsSink.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/sink/ExternalMetricsSink.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/sink/ExternalMetricsSink.java
deleted file mode 100644
index ff06307..0000000
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/sink/ExternalMetricsSink.java
+++ /dev/null
@@ -1,48 +0,0 @@
-package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.sink;
-
-import java.util.Collection;
-
-import org.apache.hadoop.metrics2.sink.timeline.TimelineMetrics;
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-public interface ExternalMetricsSink {
- /**
- * How many seconds to wait on sink before dropping metrics.
- * Note: Care should be taken that this timeout does not bottleneck the
- * sink thread.
- */
- int getSinkTimeOutSeconds();
-
- /**
- * How frequently to flush data to external system.
- * Default would be between 60 - 120 seconds, coherent with default sink
- * interval of AMS.
- */
- int getFlushSeconds();
-
- /**
- * Raw data stream to process / store on external system.
- * The data will be held in an in-memory cache and flushed at flush seconds
- * or when the cache size limit is exceeded we will flush the cache and
- * drop data if write fails.
- *
- * @param metrics {@link Collection<TimelineMetrics>}
- */
- void sinkMetricData(Collection<TimelineMetrics> metrics);
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/sink/ExternalSinkProvider.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/sink/ExternalSinkProvider.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/sink/ExternalSinkProvider.java
deleted file mode 100644
index 7c7683b..0000000
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/sink/ExternalSinkProvider.java
+++ /dev/null
@@ -1,35 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.sink;
-
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.source.InternalSourceProvider.SOURCE_NAME;
-
-
-/**
- * Configurable provider for sink classes that match the metrics sources.
- * Provider can return same sink or different sinks for each source.
- */
-public interface ExternalSinkProvider {
-
- /**
- * Return an instance of the metrics sink for the give source
- * @return {@link ExternalMetricsSink}
- */
- ExternalMetricsSink getExternalMetricsSink(SOURCE_NAME sourceName);
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/sink/HttpSinkProvider.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/sink/HttpSinkProvider.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/sink/HttpSinkProvider.java
deleted file mode 100644
index 9c2a93e..0000000
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/sink/HttpSinkProvider.java
+++ /dev/null
@@ -1,231 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.sink;
-
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.TIMELINE_METRICS_CACHE_COMMIT_INTERVAL;
-
-import java.io.BufferedReader;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.io.OutputStream;
-import java.net.HttpURLConnection;
-import java.net.URL;
-import java.security.KeyStore;
-import java.util.Collection;
-
-import javax.net.ssl.HttpsURLConnection;
-import javax.net.ssl.SSLContext;
-import javax.net.ssl.SSLSocketFactory;
-import javax.net.ssl.TrustManagerFactory;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.metrics2.sink.timeline.TimelineMetrics;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.source.InternalSourceProvider;
-import org.apache.http.client.utils.URIBuilder;
-import org.codehaus.jackson.map.AnnotationIntrospector;
-import org.codehaus.jackson.map.ObjectMapper;
-import org.codehaus.jackson.map.annotate.JsonSerialize;
-import org.codehaus.jackson.xc.JaxbAnnotationIntrospector;
-
-public class HttpSinkProvider implements ExternalSinkProvider {
- private static final Log LOG = LogFactory.getLog(HttpSinkProvider.class);
- TimelineMetricConfiguration conf = TimelineMetricConfiguration.getInstance();
-
- private String connectUrl;
- private SSLSocketFactory sslSocketFactory;
- protected static ObjectMapper mapper;
-
- static {
- mapper = new ObjectMapper();
- AnnotationIntrospector introspector = new JaxbAnnotationIntrospector();
- mapper.setAnnotationIntrospector(introspector);
- mapper.getSerializationConfig()
- .withSerializationInclusion(JsonSerialize.Inclusion.NON_NULL);
- }
-
- public HttpSinkProvider() {
- Configuration config;
- try {
- config = conf.getMetricsConf();
- } catch (Exception e) {
- throw new ExceptionInInitializerError("Unable to read configuration for sink.");
- }
- String protocol = config.get("timeline.metrics.service.external.http.sink.protocol", "http");
- String host = config.get("timeline.metrics.service.external.http.sink.host", "localhost");
- String port = config.get("timeline.metrics.service.external.http.sink.port", "6189");
-
- if (protocol.contains("https")) {
- loadTruststore(
- config.getTrimmed("timeline.metrics.service.external.http.sink.truststore.path"),
- config.getTrimmed("timeline.metrics.service.external.http.sink.truststore.type"),
- config.getTrimmed("timeline.metrics.service.external.http.sink.truststore.password")
- );
- }
-
- URIBuilder uriBuilder = new URIBuilder();
- uriBuilder.setScheme(protocol);
- uriBuilder.setHost(host);
- uriBuilder.setPort(Integer.parseInt(port));
- connectUrl = uriBuilder.toString();
- }
-
- @Override
- public ExternalMetricsSink getExternalMetricsSink(InternalSourceProvider.SOURCE_NAME sourceName) {
- return new DefaultHttpMetricsSink();
- }
-
- protected HttpURLConnection getConnection(String spec) throws IOException {
- return (HttpURLConnection) new URL(spec).openConnection();
- }
-
- // Get an ssl connection
- protected HttpsURLConnection getSSLConnection(String spec)
- throws IOException, IllegalStateException {
-
- HttpsURLConnection connection = (HttpsURLConnection) (new URL(spec).openConnection());
- connection.setSSLSocketFactory(sslSocketFactory);
- return connection;
- }
-
- protected void loadTruststore(String trustStorePath, String trustStoreType,
- String trustStorePassword) {
- if (sslSocketFactory == null) {
- if (trustStorePath == null || trustStorePassword == null) {
- String msg = "Can't load TrustStore. Truststore path or password is not set.";
- LOG.error(msg);
- throw new IllegalStateException(msg);
- }
- FileInputStream in = null;
- try {
- in = new FileInputStream(new File(trustStorePath));
- KeyStore store = KeyStore.getInstance(trustStoreType == null ?
- KeyStore.getDefaultType() : trustStoreType);
- store.load(in, trustStorePassword.toCharArray());
- TrustManagerFactory tmf = TrustManagerFactory
- .getInstance(TrustManagerFactory.getDefaultAlgorithm());
- tmf.init(store);
- SSLContext context = SSLContext.getInstance("TLS");
- context.init(null, tmf.getTrustManagers(), null);
- sslSocketFactory = context.getSocketFactory();
- } catch (Exception e) {
- LOG.error("Unable to load TrustStore", e);
- } finally {
- if (in != null) {
- try {
- in.close();
- } catch (IOException e) {
- LOG.error("Unable to load TrustStore", e);
- }
- }
- }
- }
- }
-
- class DefaultHttpMetricsSink implements ExternalMetricsSink {
-
- @Override
- public int getSinkTimeOutSeconds() {
- try {
- return conf.getMetricsConf().getInt("timeline.metrics.external.sink.http.timeout.seconds", 10);
- } catch (Exception e) {
- return 10;
- }
- }
-
- @Override
- public int getFlushSeconds() {
- try {
- return conf.getMetricsConf().getInt(TIMELINE_METRICS_CACHE_COMMIT_INTERVAL, 3);
- } catch (Exception e) {
- LOG.warn("Cannot read cache commit interval.");
- }
- return 3;
- }
-
- /**
- * Cleans up and closes an input stream
- * see http://docs.oracle.com/javase/6/docs/technotes/guides/net/http-keepalive.html
- * @param is the InputStream to clean up
- * @return string read from the InputStream
- * @throws IOException
- */
- protected String cleanupInputStream(InputStream is) throws IOException {
- StringBuilder sb = new StringBuilder();
- if (is != null) {
- try (
- InputStreamReader isr = new InputStreamReader(is);
- BufferedReader br = new BufferedReader(isr)
- ) {
- // read the response body
- String line;
- while ((line = br.readLine()) != null) {
- if (LOG.isDebugEnabled()) {
- sb.append(line);
- }
- }
- } finally {
- is.close();
- }
- }
- return sb.toString();
- }
-
- @Override
- public void sinkMetricData(Collection<TimelineMetrics> metrics) {
- HttpURLConnection connection = null;
- try {
- connection = connectUrl.startsWith("https") ? getSSLConnection(connectUrl) : getConnection(connectUrl);
-
- connection.setRequestMethod("POST");
- connection.setRequestProperty("Content-Type", "application/json");
- connection.setRequestProperty("Connection", "Keep-Alive");
- connection.setConnectTimeout(getSinkTimeOutSeconds());
- connection.setReadTimeout(getSinkTimeOutSeconds());
- connection.setDoOutput(true);
-
- if (metrics != null) {
- String jsonData = mapper.writeValueAsString(metrics);
- try (OutputStream os = connection.getOutputStream()) {
- os.write(jsonData.getBytes("UTF-8"));
- }
- }
-
- int statusCode = connection.getResponseCode();
-
- if (statusCode != 200) {
- LOG.info("Unable to POST metrics to external sink, " + connectUrl +
- ", statusCode = " + statusCode);
- } else {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Metrics posted to external sink " + connectUrl);
- }
- }
- cleanupInputStream(connection.getInputStream());
-
- } catch (IOException io) {
- LOG.warn("Unable to sink data to external system.", io);
- }
- }
- }
-}
[27/28] ambari git commit: AMBARI-22740 : Fix integration test for
HBase in branch-3.0-ams due to UUID changes. (avijayan)
Posted by av...@apache.org.
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/HBaseTimelineMetricsService.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/HBaseTimelineMetricsService.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/HBaseTimelineMetricsService.java
new file mode 100644
index 0000000..b357e0b
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/HBaseTimelineMetricsService.java
@@ -0,0 +1,592 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.metrics.core.timeline;
+
+import static org.apache.ambari.metrics.core.timeline.availability.AggregationTaskRunner.ACTUAL_AGGREGATOR_NAMES;
+
+import java.io.IOException;
+import java.net.MalformedURLException;
+import java.net.URISyntaxException;
+import java.net.UnknownHostException;
+import java.sql.SQLException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ThreadFactory;
+import java.util.concurrent.TimeUnit;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import org.apache.ambari.metrics.core.timeline.aggregators.Function;
+import org.apache.ambari.metrics.core.timeline.aggregators.TimelineMetricAggregator;
+import org.apache.ambari.metrics.core.timeline.aggregators.TimelineMetricAggregatorFactory;
+import org.apache.ambari.metrics.core.timeline.availability.MetricCollectorHAController;
+import org.apache.ambari.metrics.core.timeline.discovery.TimelineMetricHostMetadata;
+import org.apache.ambari.metrics.core.timeline.discovery.TimelineMetricMetadataKey;
+import org.apache.ambari.metrics.core.timeline.discovery.TimelineMetricMetadataManager;
+import org.apache.ambari.metrics.core.timeline.function.SeriesAggregateFunction;
+import org.apache.ambari.metrics.core.timeline.function.TimelineMetricsSeriesAggregateFunction;
+import org.apache.ambari.metrics.core.timeline.function.TimelineMetricsSeriesAggregateFunctionFactory;
+import org.apache.ambari.metrics.core.timeline.query.Condition;
+import org.apache.ambari.metrics.core.timeline.query.ConditionBuilder;
+import org.apache.ambari.metrics.core.timeline.query.PhoenixTransactSQL;
+import org.apache.ambari.metrics.core.timeline.query.TopNCondition;
+import org.apache.commons.collections.MapUtils;
+import org.apache.commons.lang.StringUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.metrics2.sink.timeline.AggregationResult;
+import org.apache.hadoop.metrics2.sink.timeline.ContainerMetric;
+import org.apache.hadoop.metrics2.sink.timeline.MetricHostAggregate;
+import org.apache.hadoop.metrics2.sink.timeline.Precision;
+import org.apache.hadoop.metrics2.sink.timeline.TimelineMetric;
+import org.apache.hadoop.metrics2.sink.timeline.TimelineMetricMetadata;
+import org.apache.hadoop.metrics2.sink.timeline.TimelineMetricWithAggregatedValues;
+import org.apache.hadoop.metrics2.sink.timeline.TimelineMetrics;
+import org.apache.hadoop.metrics2.sink.timeline.TopNConfig;
+import org.apache.hadoop.service.AbstractService;
+import org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse;
+import org.apache.ambari.metrics.core.timeline.availability.AggregationTaskRunner.AGGREGATOR_NAME;
+
+import com.google.common.collect.ArrayListMultimap;
+import com.google.common.collect.Multimap;
+
+public class HBaseTimelineMetricsService extends AbstractService implements TimelineMetricStore {
+
+ static final Log LOG = LogFactory.getLog(HBaseTimelineMetricsService.class);
+ private final TimelineMetricConfiguration configuration;
+ private TimelineMetricDistributedCache cache;
+ private PhoenixHBaseAccessor hBaseAccessor;
+ private static volatile boolean isInitialized = false;
+ private final ScheduledExecutorService watchdogExecutorService = Executors.newSingleThreadScheduledExecutor();
+ private final Map<AGGREGATOR_NAME, ScheduledExecutorService> scheduledExecutors = new HashMap<>();
+ private final ConcurrentHashMap<String, Long> postedAggregatedMap = new ConcurrentHashMap<>();
+ private TimelineMetricMetadataManager metricMetadataManager;
+ private Integer defaultTopNHostsLimit;
+ private MetricCollectorHAController haController;
+ private boolean containerMetricsDisabled = false;
+
+ /**
+ * Construct the service.
+ *
+ */
+ public HBaseTimelineMetricsService(TimelineMetricConfiguration configuration) {
+ super(HBaseTimelineMetricsService.class.getName());
+ this.configuration = configuration;
+ }
+
+ @Override
+ protected void serviceInit(Configuration conf) throws Exception {
+ super.serviceInit(conf);
+ initializeSubsystem();
+ }
+
+ private TimelineMetricDistributedCache startCacheNode() throws MalformedURLException, URISyntaxException {
+ //TODO make configurable
+ return new TimelineMetricsIgniteCache();
+ }
+
+
+ private synchronized void initializeSubsystem() {
+ if (!isInitialized) {
+ hBaseAccessor = new PhoenixHBaseAccessor(null);
+ // Initialize schema
+ hBaseAccessor.initMetricSchema();
+ // Initialize metadata from store
+ try {
+ metricMetadataManager = new TimelineMetricMetadataManager(hBaseAccessor);
+ } catch (MalformedURLException | URISyntaxException e) {
+ throw new ExceptionInInitializerError("Unable to initialize metadata manager");
+ }
+ metricMetadataManager.initializeMetadata();
+ // Initialize policies before TTL update
+ hBaseAccessor.initPoliciesAndTTL();
+ // Start HA service
+ // Start the controller
+ if (!configuration.isDistributedCollectorModeDisabled()) {
+ haController = new MetricCollectorHAController(configuration);
+ try {
+ haController.initializeHAController();
+ } catch (Exception e) {
+ LOG.error(e);
+ throw new MetricsSystemInitializationException("Unable to " +
+ "initialize HA controller", e);
+ }
+ } else {
+ LOG.info("Distributed collector mode disabled");
+ }
+
+ //Initialize whitelisting & blacklisting if needed
+ TimelineMetricsFilter.initializeMetricFilter(configuration);
+
+ Configuration metricsConf = null;
+ try {
+ metricsConf = configuration.getMetricsConf();
+ } catch (Exception e) {
+ throw new ExceptionInInitializerError("Cannot initialize configuration.");
+ }
+
+ if (configuration.isCollectorInMemoryAggregationEnabled()) {
+ try {
+ cache = startCacheNode();
+ } catch (Exception e) {
+ throw new MetricsSystemInitializationException("Unable to " +
+ "start cache node", e);
+ }
+ }
+
+ defaultTopNHostsLimit = Integer.parseInt(metricsConf.get(TimelineMetricConfiguration.DEFAULT_TOPN_HOSTS_LIMIT, "20"));
+ if (Boolean.parseBoolean(metricsConf.get(TimelineMetricConfiguration.USE_GROUPBY_AGGREGATOR_QUERIES, "true"))) {
+ LOG.info("Using group by aggregators for aggregating host and cluster metrics.");
+ }
+
+ // Start the cluster aggregator second
+ TimelineMetricAggregator secondClusterAggregator =
+ TimelineMetricAggregatorFactory.createTimelineClusterAggregatorSecond(
+ hBaseAccessor, metricsConf, metricMetadataManager, haController, cache);
+ scheduleAggregatorThread(secondClusterAggregator);
+
+ // Start the minute cluster aggregator
+ TimelineMetricAggregator minuteClusterAggregator =
+ TimelineMetricAggregatorFactory.createTimelineClusterAggregatorMinute(
+ hBaseAccessor, metricsConf, metricMetadataManager, haController);
+ scheduleAggregatorThread(minuteClusterAggregator);
+
+ // Start the hourly cluster aggregator
+ TimelineMetricAggregator hourlyClusterAggregator =
+ TimelineMetricAggregatorFactory.createTimelineClusterAggregatorHourly(
+ hBaseAccessor, metricsConf, metricMetadataManager, haController);
+ scheduleAggregatorThread(hourlyClusterAggregator);
+
+ // Start the daily cluster aggregator
+ TimelineMetricAggregator dailyClusterAggregator =
+ TimelineMetricAggregatorFactory.createTimelineClusterAggregatorDaily(
+ hBaseAccessor, metricsConf, metricMetadataManager, haController);
+ scheduleAggregatorThread(dailyClusterAggregator);
+
+ // Start the minute host aggregator
+ if (configuration.isHostInMemoryAggregationEnabled()) {
+ LOG.info("timeline.metrics.host.inmemory.aggregation is set to True, switching to filtering host minute aggregation on collector");
+ TimelineMetricAggregator minuteHostAggregator =
+ TimelineMetricAggregatorFactory.createFilteringTimelineMetricAggregatorMinute(
+ hBaseAccessor, metricsConf, metricMetadataManager, haController, postedAggregatedMap);
+ scheduleAggregatorThread(minuteHostAggregator);
+ } else {
+ TimelineMetricAggregator minuteHostAggregator =
+ TimelineMetricAggregatorFactory.createTimelineMetricAggregatorMinute(
+ hBaseAccessor, metricsConf, metricMetadataManager, haController);
+ scheduleAggregatorThread(minuteHostAggregator);
+ }
+
+ // Start the hourly host aggregator
+ TimelineMetricAggregator hourlyHostAggregator =
+ TimelineMetricAggregatorFactory.createTimelineMetricAggregatorHourly(
+ hBaseAccessor, metricsConf, metricMetadataManager, haController);
+ scheduleAggregatorThread(hourlyHostAggregator);
+
+ // Start the daily host aggregator
+ TimelineMetricAggregator dailyHostAggregator =
+ TimelineMetricAggregatorFactory.createTimelineMetricAggregatorDaily(
+ hBaseAccessor, metricsConf, metricMetadataManager, haController);
+ scheduleAggregatorThread(dailyHostAggregator);
+
+ if (!configuration.isTimelineMetricsServiceWatcherDisabled()) {
+ int initDelay = configuration.getTimelineMetricsServiceWatcherInitDelay();
+ int delay = configuration.getTimelineMetricsServiceWatcherDelay();
+ // Start the watchdog
+ watchdogExecutorService.scheduleWithFixedDelay(
+ new TimelineMetricStoreWatcher(this, configuration),
+ initDelay, delay, TimeUnit.SECONDS);
+ LOG.info("Started watchdog for timeline metrics store with initial " +
+ "delay = " + initDelay + ", delay = " + delay);
+ }
+ containerMetricsDisabled = configuration.isContainerMetricsDisabled();
+ isInitialized = true;
+ }
+
+ }
+
+ @Override
+ protected void serviceStop() throws Exception {
+ super.serviceStop();
+ }
+
+ @Override
+ public TimelineMetrics getTimelineMetrics(List<String> metricNames,
+ List<String> hostnames, String applicationId, String instanceId,
+ Long startTime, Long endTime, Precision precision, Integer limit,
+ boolean groupedByHosts, TopNConfig topNConfig, String seriesAggregateFunction) throws SQLException, IOException {
+
+ if (metricNames == null || metricNames.isEmpty()) {
+ throw new IllegalArgumentException("No metric name filter specified.");
+ }
+ if ((startTime == null && endTime != null)
+ || (startTime != null && endTime == null)) {
+ throw new IllegalArgumentException("Open ended query not supported ");
+ }
+ if (limit != null && limit > PhoenixHBaseAccessor.RESULTSET_LIMIT){
+ throw new IllegalArgumentException("Limit too big");
+ }
+
+ TimelineMetricsSeriesAggregateFunction seriesAggrFunctionInstance = null;
+ if (!StringUtils.isEmpty(seriesAggregateFunction)) {
+ SeriesAggregateFunction func = SeriesAggregateFunction.getFunction(seriesAggregateFunction);
+ seriesAggrFunctionInstance = TimelineMetricsSeriesAggregateFunctionFactory.newInstance(func);
+ }
+
+ Multimap<String, List<Function>> metricFunctions =
+ parseMetricNamesToAggregationFunctions(metricNames);
+
+ List<byte[]> uuids = metricMetadataManager.getUuids(metricFunctions.keySet(), hostnames, applicationId, instanceId);
+
+ ConditionBuilder conditionBuilder = new ConditionBuilder(new ArrayList<String>(metricFunctions.keySet()))
+ .hostnames(hostnames)
+ .appId(applicationId)
+ .instanceId(instanceId)
+ .startTime(startTime)
+ .endTime(endTime)
+ .precision(precision)
+ .limit(limit)
+ .grouped(groupedByHosts)
+ .uuid(uuids);
+
+ if (topNConfig != null) {
+ if (TopNCondition.isTopNHostCondition(metricNames, hostnames) ^ //Only 1 condition should be true.
+ TopNCondition.isTopNMetricCondition(metricNames, hostnames)) {
+ conditionBuilder.topN(topNConfig.getTopN());
+ conditionBuilder.isBottomN(topNConfig.getIsBottomN());
+ Function.ReadFunction readFunction = Function.ReadFunction.getFunction(topNConfig.getTopNFunction());
+ Function function = new Function(readFunction, null);
+ conditionBuilder.topNFunction(function);
+ } else {
+ LOG.info("Invalid Input for TopN query. Ignoring TopN Request.");
+ }
+ } else if (startTime != null && hostnames != null && hostnames.size() > defaultTopNHostsLimit) {
+ // if (timeseries query AND hostnames passed AND size(hostnames) > limit)
+ LOG.info("Requesting data for more than " + defaultTopNHostsLimit + " Hosts. " +
+ "Defaulting to Top " + defaultTopNHostsLimit);
+ conditionBuilder.topN(defaultTopNHostsLimit);
+ conditionBuilder.isBottomN(false);
+ }
+
+ Condition condition = conditionBuilder.build();
+
+ TimelineMetrics metrics;
+
+ if (hostnames == null || hostnames.isEmpty()) {
+ metrics = hBaseAccessor.getAggregateMetricRecords(condition, metricFunctions);
+ } else {
+ metrics = hBaseAccessor.getMetricRecords(condition, metricFunctions);
+ }
+
+ metrics = postProcessMetrics(metrics);
+
+ if (metrics.getMetrics().size() == 0) {
+ return metrics;
+ }
+
+ return seriesAggregateMetrics(seriesAggrFunctionInstance, metrics);
+ }
+
+ private TimelineMetrics postProcessMetrics(TimelineMetrics metrics) {
+ List<TimelineMetric> metricsList = metrics.getMetrics();
+
+ for (TimelineMetric metric : metricsList){
+ String name = metric.getMetricName();
+ if (name.contains("._rate")){
+ updateValuesAsRate(metric.getMetricValues(), false);
+ } else if (name.contains("._diff")) {
+ updateValuesAsRate(metric.getMetricValues(), true);
+ }
+ }
+
+ return metrics;
+ }
+
+ private TimelineMetrics seriesAggregateMetrics(TimelineMetricsSeriesAggregateFunction seriesAggrFuncInstance,
+ TimelineMetrics metrics) {
+ if (seriesAggrFuncInstance != null) {
+ TimelineMetric appliedMetric = seriesAggrFuncInstance.apply(metrics);
+ metrics.setMetrics(Collections.singletonList(appliedMetric));
+ }
+ return metrics;
+ }
+
+ static Map<Long, Double> updateValuesAsRate(Map<Long, Double> metricValues, boolean isDiff) {
+ Long prevTime = null;
+ Double prevVal = null;
+ long step;
+ Double diff;
+
+ for(Iterator<Map.Entry<Long, Double>> it = metricValues.entrySet().iterator(); it.hasNext(); ) {
+ Map.Entry<Long, Double> timeValueEntry = it.next();
+ Long currTime = timeValueEntry.getKey();
+ Double currVal = timeValueEntry.getValue();
+
+ if (prevTime != null) {
+ step = currTime - prevTime;
+ diff = currVal - prevVal;
+ if (diff < 0) {
+ it.remove(); //Discard calculating rate when the metric counter has been reset.
+ } else {
+ Double rate = isDiff ? diff : (diff / TimeUnit.MILLISECONDS.toSeconds(step));
+ timeValueEntry.setValue(rate);
+ }
+ } else {
+ it.remove();
+ }
+
+ prevTime = currTime;
+ prevVal = currVal;
+ }
+
+ return metricValues;
+ }
+
+ static Multimap<String, List<Function>> parseMetricNamesToAggregationFunctions(List<String> metricNames) {
+ Multimap<String, List<Function>> metricsFunctions = ArrayListMultimap.create();
+
+ for (String metricName : metricNames){
+ Function function = Function.DEFAULT_VALUE_FUNCTION;
+ String cleanMetricName = metricName;
+
+ try {
+ function = Function.fromMetricName(metricName);
+ int functionStartIndex = metricName.indexOf("._");
+ if (functionStartIndex > 0) {
+ cleanMetricName = metricName.substring(0, functionStartIndex);
+ }
+ } catch (Function.FunctionFormatException ffe){
+ // unknown function so
+ // fallback to VALUE, and fullMetricName
+ }
+
+ List<Function> functionsList = new ArrayList<>();
+ functionsList.add(function);
+ metricsFunctions.put(cleanMetricName, functionsList);
+ }
+
+ return metricsFunctions;
+ }
+
+ @Override
+ public TimelinePutResponse putMetrics(TimelineMetrics metrics) throws SQLException, IOException {
+ // Error indicated by the Sql exception
+ TimelinePutResponse response = new TimelinePutResponse();
+
+ hBaseAccessor.insertMetricRecordsWithMetadata(metricMetadataManager, metrics, false);
+
+ if (configuration.isCollectorInMemoryAggregationEnabled()) {
+ cache.putMetrics(metrics.getMetrics(), metricMetadataManager);
+ }
+
+ return response;
+ }
+
+ @Override
+ public TimelinePutResponse putContainerMetrics(List<ContainerMetric> metrics)
+ throws SQLException, IOException {
+
+ if (containerMetricsDisabled) {
+ LOG.debug("Ignoring submitted container metrics according to configuration. Values will not be stored.");
+ return new TimelinePutResponse();
+ }
+
+ hBaseAccessor.insertContainerMetrics(metrics);
+ return new TimelinePutResponse();
+ }
+
+ @Override
+ public Map<String, List<TimelineMetricMetadata>> getTimelineMetricMetadata(String appId, String metricPattern,
+ boolean includeBlacklistedMetrics) throws SQLException, IOException {
+ Map<TimelineMetricMetadataKey, TimelineMetricMetadata> metadata =
+ metricMetadataManager.getMetadataCache();
+
+ boolean filterByAppId = StringUtils.isNotEmpty(appId);
+ boolean filterByMetricName = StringUtils.isNotEmpty(metricPattern);
+ Pattern metricFilterPattern = null;
+ if (filterByMetricName) {
+ metricFilterPattern = Pattern.compile(metricPattern);
+ }
+
+ // Group Metadata by AppId
+ Map<String, List<TimelineMetricMetadata>> metadataByAppId = new HashMap<>();
+ for (TimelineMetricMetadata metricMetadata : metadata.values()) {
+
+ if (!includeBlacklistedMetrics && !metricMetadata.isWhitelisted()) {
+ continue;
+ }
+
+ String currentAppId = metricMetadata.getAppId();
+ if (filterByAppId && !currentAppId.equals(appId)) {
+ continue;
+ }
+
+ if (filterByMetricName) {
+ Matcher m = metricFilterPattern.matcher(metricMetadata.getMetricName());
+ if (!m.find()) {
+ continue;
+ }
+ }
+
+ List<TimelineMetricMetadata> metadataList = metadataByAppId.get(currentAppId);
+ if (metadataList == null) {
+ metadataList = new ArrayList<>();
+ metadataByAppId.put(currentAppId, metadataList);
+ }
+
+ metadataList.add(metricMetadata);
+ }
+
+ return metadataByAppId;
+ }
+
+ @Override
+ public byte[] getUuid(String metricName, String appId, String instanceId, String hostname) throws SQLException, IOException {
+ return metricMetadataManager.getUuid(metricName, appId, instanceId, hostname);
+ }
+
+ @Override
+ public Map<String, Set<String>> getHostAppsMetadata() throws SQLException, IOException {
+ Map<String, TimelineMetricHostMetadata> hostsMetadata = metricMetadataManager.getHostedAppsCache();
+ Map<String, Set<String>> hostAppMap = new HashMap<>();
+ for (String hostname : hostsMetadata.keySet()) {
+ hostAppMap.put(hostname, hostsMetadata.get(hostname).getHostedApps().keySet());
+ }
+ return hostAppMap;
+ }
+
+ @Override
+ public TimelinePutResponse putHostAggregatedMetrics(AggregationResult aggregationResult) throws SQLException, IOException {
+ Map<TimelineMetric, MetricHostAggregate> aggregateMap = new HashMap<>();
+ String hostname = null;
+ for (TimelineMetricWithAggregatedValues entry : aggregationResult.getResult()) {
+ aggregateMap.put(entry.getTimelineMetric(), entry.getMetricAggregate());
+ hostname = hostname == null ? entry.getTimelineMetric().getHostName() : hostname;
+ break;
+ }
+ long timestamp = aggregationResult.getTimeInMilis();
+ postedAggregatedMap.put(hostname, timestamp);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug(String.format("Adding host %s to aggregated by in-memory aggregator. Timestamp : %s", hostname, timestamp));
+ }
+ hBaseAccessor.saveHostAggregateRecords(aggregateMap, PhoenixTransactSQL.METRICS_AGGREGATE_MINUTE_TABLE_NAME);
+
+
+ return new TimelinePutResponse();
+ }
+
+ @Override
+ public Map<String, Map<String,Set<String>>> getInstanceHostsMetadata(String instanceId, String appId) throws SQLException, IOException {
+ Map<String, Set<String>> hostedApps = getHostAppsMetadata();
+ Map<String, Set<String>> instanceHosts = metricMetadataManager.getHostedInstanceCache();
+ if (configuration.getTimelineMetricsMultipleClusterSupport()) {
+ instanceHosts = metricMetadataManager.getHostedInstanceCache();
+ }
+ Map<String, Map<String, Set<String>>> instanceAppHosts = new HashMap<>();
+
+ if (MapUtils.isEmpty(instanceHosts)) {
+ Map<String, Set<String>> appHostMap = new HashMap<String, Set<String>>();
+ for (String host : hostedApps.keySet()) {
+ for (String app : hostedApps.get(host)) {
+ if (!appHostMap.containsKey(app)) {
+ appHostMap.put(app, new HashSet<String>());
+ }
+ appHostMap.get(app).add(host);
+ }
+ }
+ instanceAppHosts.put("", appHostMap);
+ } else {
+ for (String instance : instanceHosts.keySet()) {
+
+ if (StringUtils.isNotEmpty(instanceId) && !instance.equals(instanceId)) {
+ continue;
+ }
+ Map<String, Set<String>> appHostMap = new HashMap<String, Set<String>>();
+ instanceAppHosts.put(instance, appHostMap);
+
+ Set<String> hostsWithInstance = instanceHosts.get(instance);
+ for (String host : hostsWithInstance) {
+ for (String app : hostedApps.get(host)) {
+ if (StringUtils.isNotEmpty(appId) && !app.equals(appId)) {
+ continue;
+ }
+
+ if (!appHostMap.containsKey(app)) {
+ appHostMap.put(app, new HashSet<String>());
+ }
+ appHostMap.get(app).add(host);
+ }
+ }
+ }
+ }
+
+ return instanceAppHosts;
+ }
+
+ @Override
+ public List<String> getLiveInstances() {
+
+ List<String> instances = null;
+ try {
+ if (haController == null) {
+ // Always return current host as live (embedded operation mode)
+ return Collections.singletonList(configuration.getInstanceHostnameFromEnv());
+ }
+ instances = haController.getLiveInstanceHostNames();
+ if (instances == null || instances.isEmpty()) {
+ // fallback
+ instances = Collections.singletonList(configuration.getInstanceHostnameFromEnv());
+ }
+ } catch (UnknownHostException e) {
+ LOG.debug("Exception on getting hostname from env.", e);
+ }
+ return instances;
+ }
+
+ private void scheduleAggregatorThread(final TimelineMetricAggregator aggregator) {
+ if (!aggregator.isDisabled()) {
+ ScheduledExecutorService executorService = Executors.newSingleThreadScheduledExecutor(
+ new ThreadFactory() {
+ @Override
+ public Thread newThread(Runnable r) {
+ return new Thread(r, ACTUAL_AGGREGATOR_NAMES.get(aggregator.getName()));
+ }
+ }
+ );
+ scheduledExecutors.put(aggregator.getName(), executorService);
+ executorService.scheduleAtFixedRate(aggregator,
+ 0l,
+ aggregator.getSleepIntervalMillis(),
+ TimeUnit.MILLISECONDS);
+ LOG.info("Scheduled aggregator thread " + aggregator.getName() + " every " +
+ + aggregator.getSleepIntervalMillis() + " milliseconds.");
+ } else {
+ LOG.info("Skipped scheduling " + aggregator.getName() + " since it is disabled.");
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/MetricsCacheCommitterThread.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/MetricsCacheCommitterThread.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/MetricsCacheCommitterThread.java
new file mode 100644
index 0000000..d0bf137
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/MetricsCacheCommitterThread.java
@@ -0,0 +1,38 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.metrics.core.timeline;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+public class MetricsCacheCommitterThread implements Runnable {
+
+ private static final Log LOG = LogFactory.getLog(MetricsCacheCommitterThread.class);
+ private static PhoenixHBaseAccessor phoenixHBaseAccessor;
+
+ public MetricsCacheCommitterThread(PhoenixHBaseAccessor phoenixHBaseAccessor) {
+ this.phoenixHBaseAccessor = phoenixHBaseAccessor;
+ }
+ @Override
+ public void run() {
+ LOG.debug("Checking if metrics cache is empty");
+ if (!phoenixHBaseAccessor.isInsertCacheEmpty()) {
+ phoenixHBaseAccessor.commitMetricsFromCache();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/MetricsSystemInitializationException.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/MetricsSystemInitializationException.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/MetricsSystemInitializationException.java
new file mode 100644
index 0000000..f24f6e9
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/MetricsSystemInitializationException.java
@@ -0,0 +1,41 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.metrics.core.timeline;
+
+/**
+ * RuntimeException for initialization of metrics schema. It is RuntimeException
+ * since this is a not recoverable situation, and should be handled by main or
+ * service method followed by shutdown.
+ */
+public class MetricsSystemInitializationException extends RuntimeException {
+ public MetricsSystemInitializationException() {
+ }
+
+ public MetricsSystemInitializationException(String msg) {
+ super(msg);
+ }
+
+ public MetricsSystemInitializationException(Throwable t) {
+ super(t);
+ }
+
+ public MetricsSystemInitializationException(String msg, Throwable t) {
+ super(msg, t);
+ }
+
+}
[09/28] ambari git commit: AMBARI-22740 : Fix integration test for
HBase in branch-3.0-ams due to UUID changes. (avijayan)
Posted by av...@apache.org.
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TimelineWebServices.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TimelineWebServices.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TimelineWebServices.java
deleted file mode 100644
index d3da500..0000000
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TimelineWebServices.java
+++ /dev/null
@@ -1,513 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.server.applicationhistoryservice.webapp;
-
-import java.io.IOException;
-import java.sql.SQLException;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.EnumSet;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.SortedSet;
-import java.util.TreeSet;
-
-import javax.servlet.http.HttpServletRequest;
-import javax.servlet.http.HttpServletResponse;
-import javax.ws.rs.Consumes;
-import javax.ws.rs.GET;
-import javax.ws.rs.POST;
-import javax.ws.rs.Path;
-import javax.ws.rs.Produces;
-import javax.ws.rs.QueryParam;
-import javax.ws.rs.WebApplicationException;
-import javax.ws.rs.core.Context;
-import javax.ws.rs.core.MediaType;
-import javax.ws.rs.core.Response;
-import javax.xml.bind.annotation.XmlAccessType;
-import javax.xml.bind.annotation.XmlAccessorType;
-import javax.xml.bind.annotation.XmlElement;
-import javax.xml.bind.annotation.XmlRootElement;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.classification.InterfaceAudience.Public;
-import org.apache.hadoop.classification.InterfaceStability.Unstable;
-import org.apache.hadoop.metrics2.sink.timeline.AggregationResult;
-import org.apache.hadoop.metrics2.sink.timeline.ContainerMetric;
-import org.apache.hadoop.metrics2.sink.timeline.Precision;
-import org.apache.hadoop.metrics2.sink.timeline.PrecisionLimitExceededException;
-import org.apache.hadoop.metrics2.sink.timeline.TimelineMetric;
-import org.apache.hadoop.metrics2.sink.timeline.TimelineMetricMetadata;
-import org.apache.hadoop.metrics2.sink.timeline.TimelineMetrics;
-import org.apache.hadoop.metrics2.sink.timeline.TopNConfig;
-import org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricStore;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.GenericObjectMapper;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.NameValuePair;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.TimelineReader.Field;
-import org.apache.hadoop.yarn.util.timeline.TimelineUtils;
-import org.apache.hadoop.yarn.webapp.BadRequestException;
-
-import com.google.inject.Inject;
-import com.google.inject.Singleton;
-
-@Singleton
-@Path("/ws/v1/timeline")
-public class TimelineWebServices {
- private static final Log LOG = LogFactory.getLog(TimelineWebServices.class);
-
- private TimelineMetricStore timelineMetricStore;
-
- @Inject
- public TimelineWebServices(TimelineMetricStore timelineMetricStore) {
- this.timelineMetricStore = timelineMetricStore;
- }
-
- @XmlRootElement(name = "about")
- @XmlAccessorType(XmlAccessType.NONE)
- @Public
- @Unstable
- public static class AboutInfo {
-
- private String about;
-
- public AboutInfo() {
-
- }
-
- public AboutInfo(String about) {
- this.about = about;
- }
-
- @XmlElement(name = "About")
- public String getAbout() {
- return about;
- }
-
- public void setAbout(String about) {
- this.about = about;
- }
-
- }
-
- /**
- * Return the description of the timeline web services.
- */
- @GET
- @Produces({ MediaType.APPLICATION_JSON /* , MediaType.APPLICATION_XML */})
- public AboutInfo about(
- @Context HttpServletRequest req,
- @Context HttpServletResponse res) {
- init(res);
- return new AboutInfo("AMS API");
- }
-
- /**
- * Store the given metrics into the timeline store, and return errors that
- * happened during storing.
- */
- @Path("/metrics")
- @POST
- @Consumes({ MediaType.APPLICATION_JSON /* , MediaType.APPLICATION_XML */})
- public TimelinePutResponse postMetrics(
- @Context HttpServletRequest req,
- @Context HttpServletResponse res,
- TimelineMetrics metrics) {
-
- init(res);
- if (metrics == null) {
- return new TimelinePutResponse();
- }
-
- try {
-
- // TODO: Check ACLs for MetricEntity using the TimelineACLManager.
- // TODO: Save owner of the MetricEntity.
-
- if (LOG.isDebugEnabled()) {
- LOG.debug("Storing metrics: " +
- TimelineUtils.dumpTimelineRecordtoJSON(metrics, true));
- }
-
- return timelineMetricStore.putMetrics(metrics);
-
- } catch (Exception e) {
- LOG.error("Error saving metrics.", e);
- throw new WebApplicationException(e, Response.Status.INTERNAL_SERVER_ERROR);
- }
- }
-
- /**
- * Store the given metrics into the timeline store, and return errors that
- * happened during storing.
- */
- @Path("/metrics/aggregated")
- @POST
- @Consumes({ MediaType.APPLICATION_JSON /* , MediaType.APPLICATION_XML */})
- public TimelinePutResponse postAggregatedMetrics(
- @Context HttpServletRequest req,
- @Context HttpServletResponse res,
- AggregationResult metrics) {
-
- init(res);
- if (metrics == null) {
- return new TimelinePutResponse();
- }
-
- try {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Storing aggregated metrics: " +
- TimelineUtils.dumpTimelineRecordtoJSON(metrics, true));
- }
-
- return timelineMetricStore.putHostAggregatedMetrics(metrics);
- } catch (Exception e) {
- LOG.error("Error saving metrics.", e);
- throw new WebApplicationException(e, Response.Status.INTERNAL_SERVER_ERROR);
- }
- }
-
- @Path("/containermetrics")
- @POST
- @Consumes({ MediaType.APPLICATION_JSON /* , MediaType.APPLICATION_XML */})
- public TimelinePutResponse postContainerMetrics(
- @Context HttpServletRequest req,
- @Context HttpServletResponse res,
- List<ContainerMetric> metrics) {
- init(res);
- if (metrics == null || metrics.isEmpty()) {
- return new TimelinePutResponse();
- }
-
- try {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Storing container metrics: " + TimelineUtils
- .dumpTimelineRecordtoJSON(metrics, true));
- }
-
- return timelineMetricStore.putContainerMetrics(metrics);
-
- } catch (Exception e) {
- LOG.error("Error saving metrics.", e);
- throw new WebApplicationException(e, Response.Status.INTERNAL_SERVER_ERROR);
- }
- }
-
- /**
- * Query for a set of different metrics satisfying the filter criteria.
- * All query params are optional. The default limit will apply if none
- * specified.
- *
- * @param metricNames Comma separated list of metrics to retrieve.
- * @param appId Application Id for the requested metrics.
- * @param instanceId Application instance id.
- * @param hostname Hostname where the metrics originated.
- * @param startTime Start time for the metric records retrieved.
- * @param precision Precision [ seconds, minutes, hours ]
- * @param limit limit on total number of {@link TimelineMetric} records
- * retrieved.
- * @return {@link @TimelineMetrics}
- */
- @GET
- @Path("/metrics")
- @Produces({ MediaType.APPLICATION_JSON })
- public TimelineMetrics getTimelineMetrics(
- @Context HttpServletRequest req,
- @Context HttpServletResponse res,
- @QueryParam("metricNames") String metricNames,
- @QueryParam("appId") String appId,
- @QueryParam("instanceId") String instanceId,
- @QueryParam("hostname") String hostname,
- @QueryParam("startTime") String startTime,
- @QueryParam("endTime") String endTime,
- @QueryParam("precision") String precision,
- @QueryParam("limit") String limit,
- @QueryParam("grouped") String grouped,
- @QueryParam("topN") String topN,
- @QueryParam("topNFunction") String topNFunction,
- @QueryParam("isBottomN") String isBottomN,
- @QueryParam("seriesAggregateFunction") String seriesAggregateFunction
- ) {
- init(res);
- try {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Request for metrics => metricNames: " + metricNames + ", " +
- "appId: " + appId + ", instanceId: " + instanceId + ", " +
- "hostname: " + hostname + ", startTime: " + startTime + ", " +
- "endTime: " + endTime + ", " +
- "precision: " + precision + "seriesAggregateFunction: " + seriesAggregateFunction);
- }
-
- return timelineMetricStore.getTimelineMetrics(
- parseListStr(metricNames, ","), parseListStr(hostname, ","), appId, parseStr(instanceId),
- parseLongStr(startTime), parseLongStr(endTime),
- Precision.getPrecision(precision), parseIntStr(limit),
- parseBoolean(grouped), parseTopNConfig(topN, topNFunction, isBottomN),
- seriesAggregateFunction);
-
- } catch (NumberFormatException ne) {
- throw new BadRequestException("startTime and limit should be numeric " +
- "values");
- } catch (Precision.PrecisionFormatException pfe) {
- throw new BadRequestException("precision should be seconds, minutes " +
- "or hours");
- } catch (PrecisionLimitExceededException iae) {
- throw new PrecisionLimitExceededException(iae.getMessage());
- } catch (IllegalArgumentException iae) {
- throw new BadRequestException(iae.getMessage());
- } catch (SQLException | IOException e) {
- throw new WebApplicationException(e,
- Response.Status.INTERNAL_SERVER_ERROR);
- }
- }
-
- @GET
- @Path("/metrics/metadata")
- @Produces({ MediaType.APPLICATION_JSON })
- public Map<String, List<TimelineMetricMetadata>> getTimelineMetricMetadata(
- @Context HttpServletRequest req,
- @Context HttpServletResponse res,
- @QueryParam("appId") String appId,
- @QueryParam("metricName") String metricPattern,
- @QueryParam("includeAll") String includeBlacklistedMetrics
- ) {
- init(res);
-
- try {
- return timelineMetricStore.getTimelineMetricMetadata(
- parseStr(appId),
- parseStr(metricPattern),
- parseBoolean(includeBlacklistedMetrics));
- } catch (Exception e) {
- throw new WebApplicationException(e, Response.Status.INTERNAL_SERVER_ERROR);
- }
- }
-
- @GET
- @Path("/metrics/hosts")
- @Produces({ MediaType.APPLICATION_JSON })
- public Map<String, Set<String>> getHostedAppsMetadata(
- @Context HttpServletRequest req,
- @Context HttpServletResponse res
- ) {
- init(res);
-
- try {
- return timelineMetricStore.getHostAppsMetadata();
- } catch (Exception e) {
- throw new WebApplicationException(e, Response.Status.INTERNAL_SERVER_ERROR);
- }
- }
-
- @GET
- @Path("/metrics/instances")
- @Produces({ MediaType.APPLICATION_JSON })
- public Map<String, Map<String, Set<String>>> getClusterHostsMetadata(
- @Context HttpServletRequest req,
- @Context HttpServletResponse res,
- @QueryParam("appId") String appId,
- @QueryParam("instanceId") String instanceId
- ) {
- init(res);
-
- try {
- return timelineMetricStore.getInstanceHostsMetadata(instanceId, appId);
- } catch (Exception e) {
- throw new WebApplicationException(e, Response.Status.INTERNAL_SERVER_ERROR);
- }
- }
-
- @GET
- @Path("/metrics/uuid")
- @Produces({ MediaType.APPLICATION_JSON })
- public byte[] getUuid(
- @Context HttpServletRequest req,
- @Context HttpServletResponse res,
- @QueryParam("metricName") String metricName,
- @QueryParam("appId") String appId,
- @QueryParam("instanceId") String instanceId,
- @QueryParam("hostname") String hostname
- ) {
- init(res);
-
- try {
- return timelineMetricStore.getUuid(metricName, appId, instanceId, hostname);
- } catch (Exception e) {
- throw new WebApplicationException(e, Response.Status.INTERNAL_SERVER_ERROR);
- }
- }
-
- /**
- * This is a discovery endpoint that advertises known live collector
- * instances. Note: It will always answer with current instance as live.
- * This can be utilized as a liveliness pinger endpoint since the instance
- * names are cached and thereby no synchronous calls result from this API
- *
- * @return List<String> hostnames</String>
- */
- @GET
- @Path("/metrics/livenodes")
- @Produces({ MediaType.APPLICATION_JSON })
- public List<String> getLiveCollectorNodes(
- @Context HttpServletRequest req,
- @Context HttpServletResponse res
- ) {
- init(res);
-
- return timelineMetricStore.getLiveInstances();
- }
-
- private void init(HttpServletResponse response) {
- response.setContentType(null);
- }
-
- private static SortedSet<String> parseArrayStr(String str, String delimiter) {
- if (str == null) {
- return null;
- }
- SortedSet<String> strSet = new TreeSet<String>();
- String[] strs = str.split(delimiter);
- for (String aStr : strs) {
- strSet.add(aStr.trim());
- }
- return strSet;
- }
-
- private static NameValuePair parsePairStr(String str, String delimiter) {
- if (str == null) {
- return null;
- }
- String[] strs = str.split(delimiter, 2);
- try {
- return new NameValuePair(strs[0].trim(),
- GenericObjectMapper.OBJECT_READER.readValue(strs[1].trim()));
- } catch (Exception e) {
- // didn't work as an Object, keep it as a String
- return new NameValuePair(strs[0].trim(), strs[1].trim());
- }
- }
-
- private static Collection<NameValuePair> parsePairsStr(
- String str, String aDelimiter, String pDelimiter) {
- if (str == null) {
- return null;
- }
- String[] strs = str.split(aDelimiter);
- Set<NameValuePair> pairs = new HashSet<NameValuePair>();
- for (String aStr : strs) {
- pairs.add(parsePairStr(aStr, pDelimiter));
- }
- return pairs;
- }
-
- private static EnumSet<Field> parseFieldsStr(String str, String delimiter) {
- if (str == null) {
- return null;
- }
- String[] strs = str.split(delimiter);
- List<Field> fieldList = new ArrayList<Field>();
- for (String s : strs) {
- s = s.trim().toUpperCase();
- if (s.equals("EVENTS")) {
- fieldList.add(Field.EVENTS);
- } else if (s.equals("LASTEVENTONLY")) {
- fieldList.add(Field.LAST_EVENT_ONLY);
- } else if (s.equals("RELATEDENTITIES")) {
- fieldList.add(Field.RELATED_ENTITIES);
- } else if (s.equals("PRIMARYFILTERS")) {
- fieldList.add(Field.PRIMARY_FILTERS);
- } else if (s.equals("OTHERINFO")) {
- fieldList.add(Field.OTHER_INFO);
- } else {
- throw new IllegalArgumentException("Requested nonexistent field " + s);
- }
- }
- if (fieldList.size() == 0) {
- return null;
- }
- Field f1 = fieldList.remove(fieldList.size() - 1);
- if (fieldList.size() == 0) {
- return EnumSet.of(f1);
- } else {
- return EnumSet.of(f1, fieldList.toArray(new Field[fieldList.size()]));
- }
- }
-
- private static Long parseLongStr(String str) {
- return str == null ? null : Long.parseLong(str.trim());
- }
-
- private static Integer parseIntStr(String str) {
- return str == null ? null : Integer.parseInt(str.trim());
- }
-
- private static boolean parseBoolean(String booleanStr) {
- return booleanStr == null || Boolean.parseBoolean(booleanStr);
- }
-
- private static TopNConfig parseTopNConfig(String topN, String topNFunction,
- String bottomN) {
- if (topN == null || topN.isEmpty()) {
- return null;
- }
- Integer topNValue = parseIntStr(topN);
-
- if (topNValue == 0) {
- LOG.info("Invalid Input for TopN query. Ignoring TopN Request.");
- return null;
- }
-
- Boolean isBottomN = (bottomN != null && Boolean.parseBoolean(bottomN));
- return new TopNConfig(topNValue, topNFunction, isBottomN);
- }
-
- /**
- * Parses delimited string to list of strings. It skips strings that are
- * effectively empty (i.e. only whitespace).
- *
- */
- private static List<String> parseListStr(String str, String delimiter) {
- if (str == null || str.trim().isEmpty()){
- return null;
- }
-
- String[] split = str.trim().split(delimiter);
- List<String> list = new ArrayList<String>(split.length);
- for (String s : split) {
- if (!s.trim().isEmpty()){
- list.add(s);
- }
- }
-
- return list;
- }
-
- private static String parseStr(String str) {
- String trimmedInstance = (str == null) ? null : str.trim();
- if (trimmedInstance != null) {
- if (trimmedInstance.isEmpty() || trimmedInstance.equalsIgnoreCase("undefined")) {
- trimmedInstance = null;
- }
- }
- return trimmedInstance;
- }
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/python/main.py
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/python/main.py b/ambari-metrics/ambari-metrics-timelineservice/src/main/python/main.py
index b6b4e0b..5faeffc 100644
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/python/main.py
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/python/main.py
@@ -39,13 +39,13 @@ SERVER_START_CMD = \
"-cp {0} {1} " + \
"-Djava.net.preferIPv4Stack=true " \
"-Dproc_timelineserver " + \
- "org.apache.hadoop.yarn.server.applicationhistoryservice.ApplicationHistoryServer"
+ "org.apache.ambari.metrics.AMSApplicationServer"
SERVER_START_CMD_DEBUG = \
"-cp {0} {1} " + \
"-Djava.net.preferIPv4Stack=true " \
"-Dproc_timelineserver " + \
" -Xdebug -Xrunjdwp:transport=dt_socket,address=5005,server=y,suspend={2} " + \
- "org.apache.hadoop.yarn.server.applicationhistoryservice.ApplicationHistoryServer"
+ "org.apache.ambari.metrics.AMSApplicationServer"
AMC_DIE_MSG = "Ambari Metrics Collector java process died with exitcode {0}. Check {1} for more information."
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/loadsimulator/data/TestAppMetrics.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/loadsimulator/data/TestAppMetrics.java b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/loadsimulator/data/TestAppMetrics.java
new file mode 100644
index 0000000..89a5759
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/loadsimulator/data/TestAppMetrics.java
@@ -0,0 +1,134 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.metrics.core.loadsimulator.data;
+
+import org.apache.ambari.metrics.core.loadsimulator.util.Json;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.IOException;
+
+import static org.junit.Assert.assertEquals;
+
+public class TestAppMetrics {
+ private static final String SAMPLE_SINGLE_METRIC_HOST_JSON = "{\n" +
+ " \"metrics\" : [ {\n" +
+ " \"instanceid\" : \"\",\n" +
+ " \"hostname\" : \"localhost\",\n" +
+ " \"metrics\" : {\n" +
+ " \"0\" : \"5.35\",\n" +
+ " \"5000\" : \"5.35\",\n" +
+ " \"10000\" : \"5.35\",\n" +
+ " \"15000\" : \"5.35\"\n" +
+ " },\n" +
+ " \"starttime\" : \"1411663170112\",\n" +
+ " \"appid\" : \"HOST\",\n" +
+ " \"metricname\" : \"disk_free\"\n" +
+ " } ]\n" +
+ "}";
+
+ private static final String SAMPLE_TWO_METRIC_HOST_JSON = "{\n" +
+ " \"metrics\" : [ {\n" +
+ " \"instanceid\" : \"\",\n" +
+ " \"hostname\" : \"localhost\",\n" +
+ " \"metrics\" : {\n" +
+ " \"0\" : \"5.35\",\n" +
+ " \"5000\" : \"5.35\",\n" +
+ " \"10000\" : \"5.35\",\n" +
+ " \"15000\" : \"5.35\"\n" +
+ " },\n" +
+ " \"starttime\" : \"0\",\n" +
+ " \"appid\" : \"HOST\",\n" +
+ " \"metricname\" : \"disk_free\"\n" +
+ " }, {\n" +
+ " \"instanceid\" : \"\",\n" +
+ " \"hostname\" : \"localhost\",\n" +
+ " \"metrics\" : {\n" +
+ " \"0\" : \"94.0\",\n" +
+ " \"5000\" : \"94.0\",\n" +
+ " \"10000\" : \"94.0\",\n" +
+ " \"15000\" : \"94.0\"\n" +
+ " },\n" +
+ " \"starttime\" : \"0\",\n" +
+ " \"appid\" : \"HOST\",\n" +
+ " \"metricname\" : \"mem_cached\"\n" +
+ " } ]\n" +
+ "}";
+
+ private long[] timestamps;
+
+ @Before
+ public void setUp() throws Exception {
+ timestamps = new long[4];
+ timestamps[0] = 0;
+ timestamps[1] = timestamps[0] + 5000;
+ timestamps[2] = timestamps[1] + 5000;
+ timestamps[3] = timestamps[2] + 5000;
+
+ }
+
+ @Test
+ public void testHostDiskMetricsSerialization() throws IOException {
+ long timestamp = 1411663170112L;
+ AppMetrics appMetrics = new AppMetrics(new ApplicationInstance("localhost", AppID.HOST, ""), timestamp);
+
+ Metric diskFree = appMetrics.createMetric("disk_free");
+ double value = 5.35;
+
+ diskFree.putMetric(timestamps[0], Double.toString(value));
+ diskFree.putMetric(timestamps[1], Double.toString(value));
+ diskFree.putMetric(timestamps[2], Double.toString(value));
+ diskFree.putMetric(timestamps[3], Double.toString(value));
+
+ appMetrics.addMetric(diskFree);
+
+ String expected = SAMPLE_SINGLE_METRIC_HOST_JSON;
+ String s = new Json(true).serialize(appMetrics);
+
+ assertEquals("Serialized Host Metrics", expected, s);
+ }
+
+
+ @Test
+ public void testSingleHostManyMetricsSerialization() throws IOException {
+ AppMetrics appMetrics = new AppMetrics(new ApplicationInstance("localhost", AppID.HOST, ""), timestamps[0]);
+
+ Metric diskFree = appMetrics.createMetric("disk_free");
+ double value = 5.35;
+ diskFree.putMetric(timestamps[0], Double.toString(value));
+ diskFree.putMetric(timestamps[1], Double.toString(value));
+ diskFree.putMetric(timestamps[2], Double.toString(value));
+ diskFree.putMetric(timestamps[3], Double.toString(value));
+
+ appMetrics.addMetric(diskFree);
+
+ Metric memCache = appMetrics.createMetric("mem_cached");
+ double memVal = 94;
+ memCache.putMetric(timestamps[0], Double.toString(memVal));
+ memCache.putMetric(timestamps[1], Double.toString(memVal));
+ memCache.putMetric(timestamps[2], Double.toString(memVal));
+ memCache.putMetric(timestamps[3], Double.toString(memVal));
+
+ appMetrics.addMetric(memCache);
+
+ String expected = SAMPLE_TWO_METRIC_HOST_JSON;
+ String s = new Json(true).serialize(appMetrics);
+
+ assertEquals("Serialized Host Metrics", expected, s);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/loadsimulator/data/TestMetric.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/loadsimulator/data/TestMetric.java b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/loadsimulator/data/TestMetric.java
new file mode 100644
index 0000000..79e4b8f
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/loadsimulator/data/TestMetric.java
@@ -0,0 +1,80 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.metrics.core.loadsimulator.data;
+
+import org.apache.ambari.metrics.core.loadsimulator.util.Json;
+import org.junit.Test;
+
+import java.io.IOException;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.entry;
+import static org.junit.Assert.assertEquals;
+
+public class TestMetric {
+ private static final String SAMPLE_METRIC_IN_JSON = "{\n" +
+ " \"instanceid\" : \"\",\n" +
+ " \"hostname\" : \"localhost\",\n" +
+ " \"metrics\" : {\n" +
+ " \"0\" : \"5.35\",\n" +
+ " \"5000\" : \"5.35\",\n" +
+ " \"10000\" : \"5.35\",\n" +
+ " \"15000\" : \"5.35\"\n" +
+ " },\n" +
+ " \"starttime\" : \"0\",\n" +
+ " \"appid\" : \"HOST\",\n" +
+ " \"metricname\" : \"disk_free\"\n" +
+ "}";
+
+ @Test
+ public void testSerializeToJson() throws IOException {
+ Metric diskOnHostMetric = new Metric(new ApplicationInstance("localhost", AppID.HOST, ""), "disk_free", 0);
+
+ long timestamp = 0;
+ double value = 5.35;
+
+ diskOnHostMetric.putMetric(timestamp, Double.toString(value));
+ diskOnHostMetric.putMetric(timestamp + 5000, Double.toString(value));
+ diskOnHostMetric.putMetric(timestamp + 10000, Double.toString(value));
+ diskOnHostMetric.putMetric(timestamp + 15000, Double.toString(value));
+
+ String expected = SAMPLE_METRIC_IN_JSON;
+ String s = new Json(true).serialize(diskOnHostMetric);
+
+ assertEquals("Json should match", expected, s);
+ }
+
+ @Test
+ public void testDeserializeObjectFromString() throws IOException {
+ String source = SAMPLE_METRIC_IN_JSON;
+
+ Metric m = new Json().deserialize(source, Metric.class);
+
+ assertEquals("localhost", m.getHostname());
+ assertEquals("HOST", m.getAppid());
+ assertEquals("", m.getInstanceid());
+ assertEquals("disk_free", m.getMetricname());
+ assertEquals("0", m.getStarttime());
+
+ assertThat(m.getMetrics()).isNotEmpty().hasSize(4).contains(
+ entry("0", "5.35"),
+ entry("5000", "5.35"),
+ entry("10000", "5.35"),
+ entry("15000", "5.35"));
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/loadsimulator/jmetertest/jmetertest/AMSJMeterLoadTest.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/loadsimulator/jmetertest/jmetertest/AMSJMeterLoadTest.java b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/loadsimulator/jmetertest/jmetertest/AMSJMeterLoadTest.java
new file mode 100644
index 0000000..8d2d38e
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/loadsimulator/jmetertest/jmetertest/AMSJMeterLoadTest.java
@@ -0,0 +1,198 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.metrics.core.loadsimulator.jmetertest.jmetertest;
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.ambari.metrics.core.loadsimulator.MetricsLoadSimulator;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.BufferedReader;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ScheduledFuture;
+import java.util.concurrent.TimeUnit;
+
+public class AMSJMeterLoadTest {
+
+ private final static Logger LOG = LoggerFactory.getLogger(AMSJMeterLoadTest.class);
+ private static String PROPERTIES_FILE = "loadsimulator/ams-jmeter.properties";
+ private ScheduledExecutorService scheduledExecutorService = null;
+ private List<AppGetMetric> appGetMetrics;
+ private Properties amsJmeterProperties = null;
+
+ public AMSJMeterLoadTest(Map<String, String> args) {
+
+ String testType = args.get("type");
+ String userDefinedPropertiesFile = args.get("amsJmeterPropertiesFile");
+ if (null == userDefinedPropertiesFile || userDefinedPropertiesFile.isEmpty()) {
+ this.amsJmeterProperties = readProperties(PROPERTIES_FILE);
+ } else {
+ this.amsJmeterProperties = readProperties(userDefinedPropertiesFile);
+ }
+
+ if ("U".equals(testType)) { //GET metrics simulator
+ int numInstances = Integer.valueOf(amsJmeterProperties.getProperty("num-ui-instances"));
+ this.scheduledExecutorService = Executors.newScheduledThreadPool(numInstances);
+ this.appGetMetrics = initializeGetMetricsPayload(amsJmeterProperties);
+ this.runTest(numInstances);
+ } else { //PUT Metrics simulator
+ Map<String, String> mapArgs = new HashMap<String, String>();
+ mapArgs.put("hostName", (args.get("host-prefix") != null) ? args.get("host-prefix") : amsJmeterProperties.getProperty("host-prefix"));
+ mapArgs.put("minHostIndex", (args.get("min-host-index") != null) ? args.get("min-host-index") : amsJmeterProperties.getProperty("min-host-index"));
+ mapArgs.put("numberOfHosts", (args.get("num-hosts") != null) ? args.get("num-hosts") : amsJmeterProperties.getProperty("num-hosts"));
+ mapArgs.put("metricsHostName", (args.get("ams-host-port") != null) ? args.get("ams-host-port") : amsJmeterProperties.getProperty("ams-host-port"));
+ mapArgs.put("collectInterval", (args.get("collection-interval") != null) ? args.get("collection-interval") : amsJmeterProperties.getProperty("collection-interval"));
+ mapArgs.put("sendInterval", (args.get("send-interval") != null) ? args.get("send-interval") : amsJmeterProperties.getProperty("send-interval"));
+ mapArgs.put("master", (args.get("create-master") != null) ? args.get("create-master") : amsJmeterProperties.getProperty("create-master"));
+ System.out.println("AMS Load Simulation Parameters : " + mapArgs);
+ MetricsLoadSimulator.startTest(mapArgs);
+ }
+ }
+
+ public static Properties readProperties(String propertiesFile) {
+ try {
+ Properties properties = new Properties();
+ InputStream inputStream = ClassLoader.getSystemResourceAsStream(propertiesFile);
+ if (inputStream == null) {
+ inputStream = new FileInputStream(propertiesFile);
+ }
+ properties.load(inputStream);
+ return properties;
+ } catch (IOException ioEx) {
+ LOG.error("Error reading properties file for jmeter");
+ return null;
+ }
+ }
+
+ private static List<GetMetricRequestInfo> readMetricsFromFile(String app) {
+ InputStream input = null;
+ List<GetMetricRequestInfo> metricList = new ArrayList<>();
+ String fileName = "ui_metrics_def/" + app + ".dat";
+
+ try {
+ input = ClassLoader.getSystemResourceAsStream(fileName);
+ BufferedReader reader = new BufferedReader(new InputStreamReader(input));
+ String line;
+ List<String> metrics = new ArrayList<>();
+ while ((line = reader.readLine()) != null) {
+
+ if (line.startsWith("|")) {
+ boolean needsTimestamps = line.contains("startTime");
+ boolean needsHost = line.contains("hostname");
+ metricList.add(new GetMetricRequestInfo(metrics, needsTimestamps, needsHost));
+ metrics.clear();
+ } else {
+ metrics.add(line);
+ }
+ }
+ return metricList;
+ } catch (IOException e) {
+ LOG.error("Cannot read file " + fileName + " for appID " + app, e);
+ } finally {
+ if (input != null) {
+ try {
+ input.close();
+ } catch (IOException ex) {
+ }
+ }
+ }
+ return null;
+ }
+
+ private static List<AppGetMetric> initializeGetMetricsPayload(Properties amsJmeterProperties) {
+
+ List<AppGetMetric> appGetMetrics = new ArrayList<AppGetMetric>();
+ String appsToTest = amsJmeterProperties.getProperty("apps-to-test");
+ String[] apps;
+
+ if (appsToTest != null && !appsToTest.isEmpty()) {
+ apps = StringUtils.split(appsToTest, ",");
+ } else {
+ apps = new String[JmeterTestPlanTask.ClientApp.values().length];
+ int ctr = 0;
+ for (JmeterTestPlanTask.ClientApp app : JmeterTestPlanTask.ClientApp.values())
+ apps[ctr++] = app.getId();
+ }
+
+ for (String app : apps) {
+
+ int interval = Integer.valueOf(amsJmeterProperties.getProperty("get-interval"));
+ String intervalString = amsJmeterProperties.getProperty(app + "-get-interval");
+ if (intervalString != null && !intervalString.isEmpty()) {
+ interval = Integer.valueOf(intervalString);
+ }
+ appGetMetrics.add(new AppGetMetric(readMetricsFromFile(app), interval, app));
+ }
+
+ return appGetMetrics;
+ }
+
+ public void runTest(int numInstances) {
+
+ int appRefreshRate = Integer.valueOf(amsJmeterProperties.getProperty("app-refresh-rate"));
+ for (int i = 0; i < numInstances; i++) {
+ ScheduledFuture future = scheduledExecutorService.scheduleAtFixedRate(new JmeterTestPlanTask(appGetMetrics,
+ amsJmeterProperties), 0, appRefreshRate, TimeUnit.MILLISECONDS);
+ }
+ }
+
+ /**
+ * Sample Usage:
+ * java -cp "lib/*":ambari-metrics-timelineservice-2.1.1.0.jar org.apache.ambari.metrics
+ * .core.loadsimulator.jmeter.AMSJMeterLoadTest
+ * -t UI -p ambari-metrics-timelineservice/src/main/resources/jmeter/ams-jmeter.properties
+ */
+ public static void main(String[] args) {
+ Map<String, String> mapArgs = parseArgs(args);
+ new AMSJMeterLoadTest(mapArgs);
+ }
+
+ private static Map<String, String> parseArgs(String[] args) {
+ Map<String, String> mapProps = new HashMap<String, String>();
+ if (args.length == 0) {
+ printUsage();
+ throw new RuntimeException("Unexpected argument, See usage message.");
+ } else {
+ for (int i = 0; i < args.length; i += 2) {
+ String arg = args[i];
+ mapProps.put(arg.substring(1), args[i+1]);
+ }
+ }
+ return mapProps;
+ }
+
+ public static void printUsage() {
+ System.err.println("Usage: java AMSJmeterLoadTest [OPTIONS]");
+ System.err.println("Options: ");
+ System.err.println("[--t type (S=>Sink/U=>UI)] [-ams-host-port localhost:6188] [-min-host-index 2] [-host-prefix TestHost.] [-num-hosts 2] " +
+ "[-create-master true] [-collection-interval 10000 ] [-send-interval 60000 ] [-p amsJmeterPropertiesFile (Optional)]");
+ }
+
+}
+
+
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/loadsimulator/jmetertest/jmetertest/AppGetMetric.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/loadsimulator/jmetertest/jmetertest/AppGetMetric.java b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/loadsimulator/jmetertest/jmetertest/AppGetMetric.java
new file mode 100644
index 0000000..bc6428e
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/loadsimulator/jmetertest/jmetertest/AppGetMetric.java
@@ -0,0 +1,57 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.metrics.core.loadsimulator.jmetertest.jmetertest;
+
+import java.util.List;
+
+public class AppGetMetric {
+
+ private String app;
+ private int interval;
+ private List<GetMetricRequestInfo> requests;
+
+ public AppGetMetric(List<GetMetricRequestInfo> requests, int interval, String app) {
+ this.setMetricRequests(requests);
+ this.setInterval(interval);
+ this.setApp(app);
+ }
+
+ public List<GetMetricRequestInfo> getMetricRequests() {
+ return requests;
+ }
+
+ public void setMetricRequests(List<GetMetricRequestInfo> requests) {
+ this.requests = requests;
+ }
+
+ public int getInterval() {
+ return interval;
+ }
+
+ public void setInterval(int interval) {
+ this.interval = interval;
+ }
+
+ public String getApp() {
+ return app;
+ }
+
+ public void setApp(String app) {
+ this.app = app;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/loadsimulator/jmetertest/jmetertest/GetMetricRequestInfo.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/loadsimulator/jmetertest/jmetertest/GetMetricRequestInfo.java b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/loadsimulator/jmetertest/jmetertest/GetMetricRequestInfo.java
new file mode 100644
index 0000000..60ed3eb
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/loadsimulator/jmetertest/jmetertest/GetMetricRequestInfo.java
@@ -0,0 +1,61 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.metrics.core.loadsimulator.jmetertest.jmetertest;
+
+import org.apache.commons.lang.StringUtils;
+
+import java.util.List;
+
+
+public class GetMetricRequestInfo {
+
+ private String metricStringPayload;
+ private boolean needsTimestamps;
+ private boolean needsHost;
+
+ public GetMetricRequestInfo(List<String> metrics, boolean needsTimestamps, boolean needsHost) {
+
+ this.setMetricStringPayload(StringUtils.join(metrics, ","));
+ this.setNeedsTimestamps(needsTimestamps);
+ this.setNeedsHost(needsHost);
+ }
+
+ public String getMetricStringPayload() {
+ return metricStringPayload;
+ }
+
+ public void setMetricStringPayload(String metricStringPayload) {
+ this.metricStringPayload = metricStringPayload;
+ }
+
+ public boolean needsTimestamps() {
+ return needsTimestamps;
+ }
+
+ public void setNeedsTimestamps(boolean needsTimestamps) {
+ this.needsTimestamps = needsTimestamps;
+ }
+
+ public boolean needsHost() {
+ return needsHost;
+ }
+
+ public void setNeedsHost(boolean needsHost) {
+ this.needsHost = needsHost;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/loadsimulator/jmetertest/jmetertest/JmeterTestPlanTask.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/loadsimulator/jmetertest/jmetertest/JmeterTestPlanTask.java b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/loadsimulator/jmetertest/jmetertest/JmeterTestPlanTask.java
new file mode 100644
index 0000000..0590c73
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/loadsimulator/jmetertest/jmetertest/JmeterTestPlanTask.java
@@ -0,0 +1,276 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.metrics.core.loadsimulator.jmetertest.jmetertest;
+
+import org.apache.commons.io.IOUtils;
+import org.apache.jmeter.control.LoopController;
+import org.apache.jmeter.engine.StandardJMeterEngine;
+import org.apache.jmeter.protocol.http.sampler.HTTPSampler;
+import org.apache.jmeter.protocol.http.util.HTTPConstants;
+import org.apache.jmeter.reporters.ResultCollector;
+import org.apache.jmeter.reporters.Summariser;
+import org.apache.jmeter.testelement.TestElement;
+import org.apache.jmeter.testelement.TestPlan;
+import org.apache.jmeter.threads.JMeterContextService;
+import org.apache.jmeter.threads.ThreadGroup;
+import org.apache.jmeter.timers.ConstantTimer;
+import org.apache.jmeter.util.JMeterUtils;
+import org.apache.jorphan.collections.HashTree;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.lang.reflect.Field;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import java.util.Random;
+
+public class JmeterTestPlanTask implements Runnable {
+
+ private static StandardJMeterEngine jmeterEngine = null;
+ private final static Logger LOG = LoggerFactory.getLogger(JmeterTestPlanTask.class);
+ private List<AppGetMetric> appGetMetrics;
+ private Properties amsJmeterProperties;
+ private HashTree amsTestPlanTree;
+ private TestPlan amsTestPlan;
+ private static final String JMETER_HOME = "loadsimulator";
+ private static final String JMETER_PROPERTIES_FILE = JMETER_HOME + "/jmeter.properties";
+ private static final String SAVESERVICE_PROPERTIES_FILE = JMETER_HOME + "/saveservice.properties";
+
+ public enum ClientApp {
+ HOST("HOST"),
+ NAMENODE("NAMENODE"),
+ HBASE("HBASE"),
+ NIMBUS("NIMBUS"),
+ KAFKA_BROKER("KAFKA_BROKER"),
+ FLUME_HANDLER("FLUME_HANDLER"),
+ AMS_HBASE("AMS-HBASE"),
+ NODEMANAGER("NODEMANAGER"),
+ RESOURCEMANAGER("RESOURCEMANAGER"),
+ DATANODE("DATANODE");
+
+ private String id;
+
+ private ClientApp(String id) {
+ this.id = id;
+ }
+
+ public String getId() {
+ return id;
+ }
+ }
+
+ public JmeterTestPlanTask(List<AppGetMetric> appGetMetrics, Properties amsJmeterProperties) {
+ this.appGetMetrics = appGetMetrics;
+ this.amsJmeterProperties = amsJmeterProperties;
+ amsTestPlanTree = new HashTree();
+ amsTestPlan = new TestPlan("AMS JMeter Load Test plan");
+ System.out.println("Starting AMS Jmeter load testing");
+ }
+
+ public void run() {
+ if (jmeterEngine != null) {
+
+ Object[] threadGroups = amsTestPlanTree.getArray(amsTestPlan);
+ for (Object threadGroupObj : threadGroups) {
+ if (threadGroupObj instanceof ThreadGroup) {
+ ThreadGroup threadGroup = (ThreadGroup) threadGroupObj;
+ threadGroup.stop();
+ }
+ }
+ amsTestPlanTree.clear();
+ jmeterEngine.askThreadsToStop();
+ jmeterEngine.stopTest();
+ JMeterContextService.endTest();
+ }
+
+ //Start the new test plan for the new app.
+ try {
+ //Initialize Jmeter essentials
+ jmeterEngine = new StandardJMeterEngine();
+ JMeterContextService.getContext().setEngine(jmeterEngine);
+
+ //Workaround to supply JMeterUtils with jmeter.prooperties from JAR.
+ JMeterUtils.setJMeterHome("");
+ Field f = new JMeterUtils().getClass().getDeclaredField("appProperties");
+ f.setAccessible(true);
+ f.set(null, AMSJMeterLoadTest.readProperties(JMETER_PROPERTIES_FILE));
+
+ //Copy saveservices.properties file to tmp dir for JMeter to consume.
+ InputStream inputStream = ClassLoader.getSystemResourceAsStream(SAVESERVICE_PROPERTIES_FILE);
+ if (inputStream == null) {
+ inputStream = new FileInputStream(SAVESERVICE_PROPERTIES_FILE);
+ }
+ String tmpDir = System.getProperty("java.io.tmpdir");
+ OutputStream outputStream = new FileOutputStream(tmpDir + "/saveservice.properties");
+ IOUtils.copy(inputStream, outputStream);
+ outputStream.close();
+ JMeterUtils.setProperty("saveservice_properties", tmpDir + "/saveservice.properties");
+
+ //Initialize Test plan
+ amsTestPlan.setProperty(TestElement.TEST_CLASS, TestPlan.class.getName());
+ amsTestPlanTree.add("AMS Test plan", amsTestPlan);
+
+ //Choose a random APP to run the perform GET metrics request.
+ int currentAppIndex = new Random().nextInt(appGetMetrics.size());
+
+ //Create ThreadGroup for the App
+ createThreadGroupHashTree(currentAppIndex, amsJmeterProperties, amsTestPlanTree, amsTestPlan);
+
+ //Geneates the JMX file that you can use through the GUI mode.
+ //SaveService.saveTree(amsTestPlanTree, new FileOutputStream(JMETER_HOME + "/" + "amsTestPlan.jmx"));
+
+ //Summarizer output to get test progress in stdout like.
+ Summariser summariser = null;
+ String summariserName = JMeterUtils.getPropDefault("summariser.name", "summary");
+ if (summariserName.length() > 0) {
+ summariser = new Summariser(summariserName);
+ }
+
+ //Store execution results into a .jtl file
+ String jmeterLogFile = tmpDir + "/amsJmeterTestResults.jtl";
+ ResultCollector resultCollector = new ResultCollector(summariser);
+ resultCollector.setFilename(jmeterLogFile);
+ amsTestPlanTree.add(amsTestPlanTree.getArray()[0], resultCollector);
+ jmeterEngine.configure(amsTestPlanTree);
+ jmeterEngine.run();
+
+ LOG.info("AMS Jmeter Test started up successfully");
+
+ } catch (Exception ioEx) {
+ amsTestPlanTree.clear();
+ jmeterEngine.askThreadsToStop();
+ jmeterEngine.stopTest();
+ JMeterContextService.endTest();
+ LOG.error("Error occurred while running AMS load test : " + ioEx.getMessage());
+ ioEx.printStackTrace();
+ }
+ }
+
+ private ConstantTimer createConstantTimer(int delay) {
+ ConstantTimer timer = new ConstantTimer();
+ timer.setDelay("" + delay);
+ return timer;
+ }
+
+ private Map<String, String> getAppSpecificParameters(String app, GetMetricRequestInfo request, Properties amsJmeterProperties) {
+
+ Map<String, String> parametersMap = new HashMap<String, String>();
+ String hostPrefix = amsJmeterProperties.getProperty("host-prefix");
+ String hostSuffix = amsJmeterProperties.getProperty("host-suffix");
+ int minHostIndex = Integer.valueOf(amsJmeterProperties.getProperty("min-host-index"));
+ int numHosts = Integer.valueOf(amsJmeterProperties.getProperty("num-hosts"));
+
+ parametersMap.put("appId", app);
+
+ if (request.needsTimestamps()) {
+ long currentTime = System.currentTimeMillis();
+ long oneHourBack = currentTime - 3600 * 1000;
+ parametersMap.put("startTime", String.valueOf(oneHourBack));
+ parametersMap.put("endTime", String.valueOf(currentTime));
+ }
+
+ if (request.needsHost()) {
+ if (ClientApp.AMS_HBASE.getId().equals(app)) {
+ parametersMap.put("hostname", amsJmeterProperties.getProperty("ams-host"));
+ } else if (ClientApp.HOST.getId().equals(app) || ClientApp.NODEMANAGER.getId().equals(app)) {
+ int randomHost = minHostIndex + new Random().nextInt(numHosts);
+ parametersMap.put("hostname", hostPrefix + randomHost + hostSuffix);
+ } else {
+ parametersMap.put("hostname", hostPrefix + amsJmeterProperties.getProperty(app + "-host") + hostSuffix);
+ }
+ }
+ parametersMap.put("metricNames", request.getMetricStringPayload());
+ return parametersMap;
+ }
+
+ private void createThreadGroupHashTree(int appIndex, Properties amsJmeterProperties, HashTree amsTestPlanTree, TestPlan amsTestPlan) {
+
+ AppGetMetric appGetMetric = appGetMetrics.get(appIndex);
+ String app = appGetMetric.getApp();
+ int interval = appGetMetric.getInterval();
+
+ //Read and validate AMS information.
+ String[] amsHostPort = amsJmeterProperties.getProperty("ams-host-port").split(":");
+ String amsHost = amsHostPort[0];
+ String amsPath = amsJmeterProperties.getProperty("ams-path");
+ int amsPort = Integer.valueOf(amsHostPort[1]);
+ int numLoops = Integer.valueOf(amsJmeterProperties.getProperty("num-get-calls-per-app"));
+
+ LoopController loopController = createLoopController(app + " GET loop controller", numLoops, false);
+ for (GetMetricRequestInfo request : appGetMetric.getMetricRequests()) {
+
+ ThreadGroup threadGroup = createThreadGroup(app + " GET threadGroup", 1, 0, loopController);
+
+ HashTree threadGroupHashTree = amsTestPlanTree.add(amsTestPlan, threadGroup);
+ Map<String, String> parametersMap = getAppSpecificParameters(app, request, amsJmeterProperties);
+
+ HTTPSampler sampler = createGetSampler("GET " + app + " metrics", amsHost, amsPort, amsPath, null, parametersMap);
+
+ if (numLoops > 1) {
+ threadGroupHashTree.add(createConstantTimer(interval));
+ }
+
+ threadGroupHashTree.add(sampler);
+ }
+ }
+
+ private HTTPSampler createGetSampler(String name, String domain, int port, String path, String encoding, Map<String, String> parameters) {
+
+ HTTPSampler sampler = new HTTPSampler();
+ sampler.setDomain(domain);
+ sampler.setPort(port);
+ sampler.setPath(path);
+ sampler.setMethod(HTTPConstants.GET);
+
+ if (encoding != null)
+ sampler.setContentEncoding(encoding);
+
+ for (Map.Entry<String, String> entry : parameters.entrySet()) {
+ sampler.addArgument(entry.getKey(), entry.getValue());
+ }
+ sampler.setName(name);
+ return sampler;
+ }
+
+ private LoopController createLoopController(String name, int numLoops, boolean continueForever) {
+ LoopController loopController = new LoopController();
+ loopController.setLoops(numLoops);
+ loopController.setProperty(TestElement.TEST_CLASS, LoopController.class.getName());
+ loopController.initialize();
+ loopController.setContinueForever(continueForever);
+ loopController.setName(name);
+ return loopController;
+ }
+
+ private ThreadGroup createThreadGroup(String name, int numThreads, int rampUp, LoopController loopController) {
+ ThreadGroup threadGroup = new ThreadGroup();
+ threadGroup.setName(name);
+ threadGroup.setNumThreads(numThreads);
+ threadGroup.setRampUp(rampUp);
+ threadGroup.setSamplerController(loopController);
+ threadGroup.setProperty(TestElement.TEST_CLASS, ThreadGroup.class.getName());
+ return threadGroup;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/loadsimulator/net/TestRestMetricsSender.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/loadsimulator/net/TestRestMetricsSender.java b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/loadsimulator/net/TestRestMetricsSender.java
new file mode 100644
index 0000000..9c8e641
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/loadsimulator/net/TestRestMetricsSender.java
@@ -0,0 +1,75 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.metrics.core.loadsimulator.net;
+
+import org.junit.Test;
+
+import java.io.IOException;
+
+import static org.easymock.EasyMock.*;
+import static org.junit.Assert.assertEquals;
+
+public class TestRestMetricsSender {
+
+ @Test
+ public void testPushMetrics() throws Exception {
+ final UrlService svcMock = createStrictMock(UrlService.class);
+ final String payload = "test";
+ final String expectedResponse = "mockResponse";
+
+ expect(svcMock.send(anyString())).andReturn(expectedResponse);
+ svcMock.disconnect();
+ expectLastCall();
+
+ replay(svcMock);
+
+ RestMetricsSender sender = new RestMetricsSender("expectedHostName") {
+ @Override
+ protected UrlService getConnectedUrlService() throws IOException {
+ return svcMock;
+ }
+ };
+ String response = sender.pushMetrics(payload);
+
+ verify(svcMock);
+ assertEquals("", expectedResponse, response);
+ }
+
+ @Test
+ public void testPushMetricsFailed() throws Exception {
+ final UrlService svcMock = createStrictMock(UrlService.class);
+ final String payload = "test";
+ final String expectedResponse = "mockResponse";
+ RestMetricsSender sender = new RestMetricsSender("expectedHostName") {
+ @Override
+ protected UrlService getConnectedUrlService() throws IOException {
+ return svcMock;
+ }
+ };
+
+ expect(svcMock.send(anyString())).andThrow(new IOException());
+ svcMock.disconnect();
+ expectLastCall();
+
+ replay(svcMock);
+
+ String response = sender.pushMetrics(payload);
+
+ verify(svcMock);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/loadsimulator/net/TestStdOutMetricsSender.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/loadsimulator/net/TestStdOutMetricsSender.java b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/loadsimulator/net/TestStdOutMetricsSender.java
new file mode 100644
index 0000000..29ebda4
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/loadsimulator/net/TestStdOutMetricsSender.java
@@ -0,0 +1,37 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.metrics.core.loadsimulator.net;
+
+
+import org.junit.Test;
+
+import java.io.ByteArrayOutputStream;
+import java.io.PrintStream;
+
+public class TestStdOutMetricsSender {
+
+ @Test
+ public void testPushMetrics() throws Exception {
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ PrintStream out = new PrintStream(baos);
+ StdOutMetricsSender sender = new StdOutMetricsSender("expectedHostName", out);
+ sender.pushMetrics("test");
+
+ System.out.println(baos.toString());
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/loadsimulator/util/TestRandomMetricsProvider.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/loadsimulator/util/TestRandomMetricsProvider.java b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/loadsimulator/util/TestRandomMetricsProvider.java
new file mode 100644
index 0000000..a1801b0
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/loadsimulator/util/TestRandomMetricsProvider.java
@@ -0,0 +1,36 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.metrics.core.loadsimulator.util;
+
+import org.junit.Test;
+
+import static org.junit.Assert.assertTrue;
+
+public class TestRandomMetricsProvider {
+
+ @Test
+ public void testReturnSingle() {
+ double from = 5.25;
+ double to = 5.40;
+ RandomMetricsProvider provider = new RandomMetricsProvider(from, to);
+ double metric = provider.next();
+
+ assertTrue("Generated metric should be in range", from < metric && metric < to);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/loadsimulator/util/TestTimeStampProvider.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/loadsimulator/util/TestTimeStampProvider.java b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/loadsimulator/util/TestTimeStampProvider.java
new file mode 100644
index 0000000..9011e75
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/loadsimulator/util/TestTimeStampProvider.java
@@ -0,0 +1,51 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.metrics.core.loadsimulator.util;
+
+import org.junit.Test;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.junit.Assert.assertEquals;
+
+public class TestTimeStampProvider {
+
+ @Test
+ public void testReturnSingle() {
+ long startTime = 1411663170112L;
+ int timeStep = 5000;
+ TimeStampProvider tm = new TimeStampProvider(startTime, timeStep, 0);
+
+ long tStamp = tm.next();
+
+ assertEquals("First generated timestamp should match starttime", startTime, tStamp);
+ }
+
+ @Test
+ public void testReturnTstampsForSendInterval() throws Exception {
+ long startTime = 0;
+ int collectInterval = 5;
+ int sendInterval = 30;
+ TimeStampProvider tsp = new TimeStampProvider(startTime, collectInterval, sendInterval);
+
+ long[] timestamps = tsp.timestampsForNextInterval();
+
+ assertThat(timestamps)
+ .hasSize(6)
+ .containsOnly(0, 5, 10, 15, 20, 25);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/timeline/AbstractMiniHBaseClusterTest.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/timeline/AbstractMiniHBaseClusterTest.java b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/timeline/AbstractMiniHBaseClusterTest.java
new file mode 100644
index 0000000..0553d4c
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/timeline/AbstractMiniHBaseClusterTest.java
@@ -0,0 +1,293 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.metrics.core.timeline;
+
+import static org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguration.OUT_OFF_BAND_DATA_TIME_ALLOWANCE;
+import static org.apache.ambari.metrics.core.timeline.query.PhoenixTransactSQL.METRICS_RECORD_TABLE_NAME;
+import static org.apache.ambari.metrics.core.timeline.query.PhoenixTransactSQL.UPSERT_METRICS_SQL;
+import static org.apache.phoenix.end2end.ParallelStatsDisabledIT.tearDownMiniCluster;
+import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES;
+import static org.assertj.core.api.Assertions.assertThat;
+
+import java.io.IOException;
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.IntegrationTestingUtility;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.metrics2.sink.timeline.TimelineMetric;
+import org.apache.hadoop.metrics2.sink.timeline.TimelineMetrics;
+import org.apache.ambari.metrics.core.timeline.aggregators.AggregatorUtils;
+import org.apache.ambari.metrics.core.timeline.discovery.TimelineMetricMetadataManager;
+import org.apache.ambari.metrics.core.timeline.query.PhoenixConnectionProvider;
+import org.apache.hadoop.yarn.util.timeline.TimelineUtils;
+import org.apache.log4j.Level;
+import org.apache.log4j.Logger;
+import org.apache.phoenix.hbase.index.write.IndexWriterUtils;
+import org.apache.phoenix.query.BaseTest;
+import org.apache.phoenix.query.QueryServices;
+import org.apache.phoenix.util.PropertiesUtil;
+import org.apache.phoenix.util.ReadOnlyProps;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+public abstract class AbstractMiniHBaseClusterTest extends BaseTest {
+
+ protected static final long BATCH_SIZE = 3;
+ protected Connection conn;
+ protected PhoenixHBaseAccessor hdb;
+ protected TimelineMetricMetadataManager metadataManager;
+
+ public final Log LOG;
+
+ public AbstractMiniHBaseClusterTest() {
+ LOG = LogFactory.getLog(this.getClass());
+ }
+
+ @BeforeClass
+ public static void doSetup() throws Exception {
+ Map<String, String> props = getDefaultProps();
+ props.put(IntegrationTestingUtility.IS_DISTRIBUTED_CLUSTER, "false");
+ props.put(QueryServices.QUEUE_SIZE_ATTRIB, Integer.toString(5000));
+ props.put(IndexWriterUtils.HTABLE_THREAD_KEY, Integer.toString(100));
+ // Make a small batch size to test multiple calls to reserve sequences
+ props.put(QueryServices.SEQUENCE_CACHE_SIZE_ATTRIB, Long.toString(BATCH_SIZE));
+ // Must update config before starting server
+ setUpTestDriver(new ReadOnlyProps(props.entrySet().iterator()));
+ }
+
+ @AfterClass
+ public static void doTeardown() throws Exception {
+ dropNonSystemTables();
+ tearDownMiniCluster();
+ }
+
+ @Before
+ public void setUp() throws Exception {
+ Logger.getLogger("org.apache.ambari.metrics.core.timeline").setLevel(Level.DEBUG);
+ hdb = createTestableHBaseAccessor();
+ // inits connection, starts mini cluster
+ conn = getConnection(getUrl());
+
+ hdb.initMetricSchema();
+ metadataManager = new TimelineMetricMetadataManager(new Configuration(), hdb);
+ hdb.setMetadataInstance(metadataManager);
+ }
+
+ private void deleteTableIgnoringExceptions(Statement stmt, String tableName) {
+ try {
+ stmt.execute("delete from " + tableName);
+ } catch (Exception e) {
+ LOG.warn("Exception on delete table " + tableName, e);
+ }
+ }
+
+ @After
+ public void tearDown() {
+ Connection conn = null;
+ Statement stmt = null;
+ try {
+ conn = getConnection(getUrl());
+ stmt = conn.createStatement();
+
+ deleteTableIgnoringExceptions(stmt, "METRIC_AGGREGATE");
+ deleteTableIgnoringExceptions(stmt, "METRIC_AGGREGATE_MINUTE");
+ deleteTableIgnoringExceptions(stmt, "METRIC_AGGREGATE_HOURLY");
+ deleteTableIgnoringExceptions(stmt, "METRIC_AGGREGATE_DAILY");
+ deleteTableIgnoringExceptions(stmt, "METRIC_RECORD");
+ deleteTableIgnoringExceptions(stmt, "METRIC_RECORD_MINUTE");
+ deleteTableIgnoringExceptions(stmt, "METRIC_RECORD_HOURLY");
+ deleteTableIgnoringExceptions(stmt, "METRIC_RECORD_DAILY");
+ deleteTableIgnoringExceptions(stmt, "METRICS_METADATA");
+ deleteTableIgnoringExceptions(stmt, "HOSTED_APPS_METADATA");
+
+ conn.commit();
+ } catch (Exception e) {
+ LOG.warn("Error on deleting HBase schema.", e);
+ } finally {
+ if (stmt != null) {
+ try {
+ stmt.close();
+ } catch (SQLException e) {
+ // Ignore
+ }
+ }
+
+ if (conn != null) {
+ try {
+ conn.close();
+ } catch (SQLException e) {
+ // Ignore
+ }
+ }
+ }
+ }
+
+ public static Map<String, String> getDefaultProps() {
+ Map<String, String> props = new HashMap<String, String>();
+ // Must update config before starting server
+ props.put(QueryServices.STATS_USE_CURRENT_TIME_ATTRIB,
+ Boolean.FALSE.toString());
+ props.put("java.security.krb5.realm", "");
+ props.put("java.security.krb5.kdc", "");
+ return props;
+ }
+
+ protected Connection getConnection(String url) throws SQLException {
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ Connection conn = DriverManager.getConnection(getUrl(), props);
+ return conn;
+ }
+
+ /**
+ * A canary test. Will show if the infrastructure is set-up correctly.
+ */
+ @Test
+ public void testClusterOK() throws Exception {
+ Connection conn = getConnection(getUrl());
+ conn.setAutoCommit(true);
+
+ String sampleDDL = "CREATE TABLE TEST_METRICS " +
+ "(TEST_COLUMN VARCHAR " +
+ "CONSTRAINT pk PRIMARY KEY (TEST_COLUMN)) " +
+ "DATA_BLOCK_ENCODING='FAST_DIFF', IMMUTABLE_ROWS=true, " +
+ "TTL=86400, COMPRESSION='NONE' ";
+
+ Statement stmt = conn.createStatement();
+ stmt.executeUpdate(sampleDDL);
+ conn.commit();
+
+ ResultSet rs = stmt.executeQuery(
+ "SELECT COUNT(TEST_COLUMN) FROM TEST_METRICS");
+
+ rs.next();
+ long l = rs.getLong(1);
+ assertThat(l).isGreaterThanOrEqualTo(0);
+
+ stmt.execute("DROP TABLE TEST_METRICS");
+ conn.close();
+ }
+
+ protected PhoenixHBaseAccessor createTestableHBaseAccessor() {
+ Configuration metricsConf = new Configuration();
+ metricsConf.set(TimelineMetricConfiguration.HBASE_COMPRESSION_SCHEME, "NONE");
+ // Unit tests insert values into the future
+ metricsConf.setLong(OUT_OFF_BAND_DATA_TIME_ALLOWANCE, 600000);
+
+ return
+ new PhoenixHBaseAccessor(new TimelineMetricConfiguration(new Configuration(), metricsConf),
+ new PhoenixConnectionProvider() {
+ @Override
+ public Admin getHBaseAdmin() throws IOException {
+ try {
+ return driver.getConnectionQueryServices(null, null).getAdmin();
+ } catch (SQLException e) {
+ LOG.error(e);
+ }
+ return null;
+ }
+
+ @Override
+ public Connection getConnection() {
+ Connection connection = null;
+ try {
+ connection = DriverManager.getConnection(getUrl());
+ } catch (SQLException e) {
+ LOG.warn("Unable to connect to HBase store using Phoenix.", e);
+ }
+ return connection;
+ }
+
+ });
+ }
+
+ protected void insertMetricRecords(Connection conn, TimelineMetrics metrics)
+ throws SQLException, IOException {
+
+ List<TimelineMetric> timelineMetrics = metrics.getMetrics();
+ if (timelineMetrics == null || timelineMetrics.isEmpty()) {
+ LOG.debug("Empty metrics insert request.");
+ return;
+ }
+
+ PreparedStatement metricRecordStmt = null;
+
+ try {
+ metricRecordStmt = conn.prepareStatement(String.format(
+ UPSERT_METRICS_SQL, METRICS_RECORD_TABLE_NAME));
+
+ for (TimelineMetric metric : timelineMetrics) {
+ metricRecordStmt.clearParameters();
+
+ if (LOG.isTraceEnabled()) {
+ LOG.trace("host: " + metric.getHostName() + ", " +
+ "metricName = " + metric.getMetricName() + ", " +
+ "values: " + metric.getMetricValues());
+ }
+ double[] aggregates = AggregatorUtils.calculateAggregates(
+ metric.getMetricValues());
+
+ byte[] uuid = metadataManager.getUuid(metric);
+ if (uuid == null) {
+ LOG.error("Error computing UUID for metric. Cannot write metrics : " + metric.toString());
+ continue;
+ }
+ metricRecordStmt.setBytes(1, uuid);
+ metricRecordStmt.setLong(2, metric.getStartTime());
+ metricRecordStmt.setDouble(3, aggregates[0]);
+ metricRecordStmt.setDouble(4, aggregates[1]);
+ metricRecordStmt.setDouble(5, aggregates[2]);
+ metricRecordStmt.setInt(6, (int) aggregates[3]);
+ String json = TimelineUtils.dumpTimelineRecordtoJSON(metric.getMetricValues());
+ metricRecordStmt.setString(7, json);
+
+ try {
+ int row = metricRecordStmt.executeUpdate();
+ LOG.info("Inserted " + row + " rows.");
+ } catch (SQLException sql) {
+ LOG.error(sql);
+ }
+ }
+
+ conn.commit();
+
+ } finally {
+ if (metricRecordStmt != null) {
+ try {
+ metricRecordStmt.close();
+ } catch (SQLException e) {
+ // Ignore
+ }
+ }
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/timeline/AbstractPhoenixConnectionlessTest.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/timeline/AbstractPhoenixConnectionlessTest.java b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/timeline/AbstractPhoenixConnectionlessTest.java
new file mode 100644
index 0000000..d3fc50f
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/timeline/AbstractPhoenixConnectionlessTest.java
@@ -0,0 +1,111 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.metrics.core.timeline;
+
+import org.apache.phoenix.jdbc.PhoenixEmbeddedDriver;
+import org.apache.phoenix.jdbc.PhoenixTestDriver;
+import org.apache.phoenix.query.BaseTest;
+import org.apache.phoenix.util.PropertiesUtil;
+import org.apache.phoenix.util.ReadOnlyProps;
+import org.apache.phoenix.util.TestUtil;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.PreparedStatement;
+import static org.apache.phoenix.util.PhoenixRuntime.TENANT_ID_ATTRIB;
+import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+
+public abstract class AbstractPhoenixConnectionlessTest extends BaseTest {
+
+ protected static String getUrl() {
+ return TestUtil.PHOENIX_CONNECTIONLESS_JDBC_URL;
+ }
+
+ protected static String getUrl(String tenantId) {
+ return getUrl() + ';' + TENANT_ID_ATTRIB + '=' + tenantId;
+ }
+
+ protected static PhoenixTestDriver driver;
+
+ private static void startServer(String url) throws Exception {
+ assertNull(driver);
+ // only load the test driver if we are testing locally - for integration tests, we want to
+ // test on a wider scale
+ if (PhoenixEmbeddedDriver.isTestUrl(url)) {
+ driver = initDriver(ReadOnlyProps.EMPTY_PROPS);
+ assertTrue(DriverManager.getDriver(url) == driver);
+ driver.connect(url, PropertiesUtil.deepCopy(TEST_PROPERTIES));
+ }
+ }
+
+ protected static synchronized PhoenixTestDriver initDriver(ReadOnlyProps props) throws Exception {
+ if (driver == null) {
+ driver = new PhoenixTestDriver(props);
+ DriverManager.registerDriver(driver);
+ }
+ return driver;
+ }
+
+ private String connUrl;
+
+ @Before
+ public void setup() throws Exception {
+ connUrl = getUrl();
+ startServer(connUrl);
+ }
+
+ @Test
+ public void testStorageSystemInitialized() throws Exception {
+ String sampleDDL = "CREATE TABLE TEST_METRICS (TEST_COLUMN VARCHAR " +
+ "CONSTRAINT pk PRIMARY KEY (TEST_COLUMN)) DATA_BLOCK_ENCODING='FAST_DIFF', " +
+ "IMMUTABLE_ROWS=true, TTL=86400, COMPRESSION='SNAPPY'";
+
+ Connection conn = null;
+ PreparedStatement stmt = null;
+ try {
+ conn = DriverManager.getConnection(connUrl);
+ stmt = conn.prepareStatement(sampleDDL);
+ stmt.execute();
+ conn.commit();
+ } finally {
+ if (stmt != null) {
+ stmt.close();
+ }
+ if (conn != null) {
+ conn.close();
+ }
+ }
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ if (driver != null) {
+ try {
+ driver.close();
+ } finally {
+ PhoenixTestDriver phoenixTestDriver = driver;
+ driver = null;
+ DriverManager.deregisterDriver(phoenixTestDriver);
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/timeline/FunctionTest.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/timeline/FunctionTest.java b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/timeline/FunctionTest.java
new file mode 100644
index 0000000..03e39f7
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/timeline/FunctionTest.java
@@ -0,0 +1,62 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.metrics.core.timeline;
+
+import org.apache.ambari.metrics.core.timeline.aggregators.Function;
+import org.junit.Ignore;
+import org.junit.Test;
+
+import static org.apache.ambari.metrics.core.timeline.aggregators.Function.fromMetricName;
+import static org.apache.ambari.metrics.core.timeline.aggregators.Function.ReadFunction.AVG;
+import static org.apache.ambari.metrics.core.timeline.aggregators.Function.PostProcessingFunction.RATE;
+import static org.apache.ambari.metrics.core.timeline.aggregators.Function.PostProcessingFunction.DIFF;
+import static org.assertj.core.api.Assertions.assertThat;
+
+public class FunctionTest {
+
+ @Test
+ public void testCreation() throws Exception {
+ Function f = fromMetricName("Metric._avg");
+ assertThat(f).isEqualTo(new Function(AVG, null));
+
+ f = fromMetricName("Metric._rate._avg");
+ assertThat(f).isEqualTo(new Function(AVG, RATE));
+
+ f = fromMetricName("bytes_in");
+ assertThat(f).isEqualTo(Function.DEFAULT_VALUE_FUNCTION);
+
+ // Rate support without aggregates
+ f = fromMetricName("Metric._rate");
+ assertThat(f).isEqualTo(new Function(null, RATE));
+
+ // Diff support
+ f = fromMetricName("Metric._diff._avg");
+ assertThat(f).isEqualTo(new Function(AVG, DIFF));
+
+ // Diff support without aggregates
+ f = fromMetricName("Metric._diff");
+ assertThat(f).isEqualTo(new Function(null, DIFF));
+
+ }
+
+ @Ignore // If unknown function: behavior is best effort query without function
+ @Test(expected = Function.FunctionFormatException.class)
+ public void testNotAFunction() throws Exception {
+ fromMetricName("bytes._not._afunction");
+ }
+}
[22/28] ambari git commit: AMBARI-22740 : Fix integration test for
HBase in branch-3.0-ams due to UUID changes. (avijayan)
Posted by av...@apache.org.
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/discovery/TimelineMetricMetadataManager.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/discovery/TimelineMetricMetadataManager.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/discovery/TimelineMetricMetadataManager.java
new file mode 100644
index 0000000..97eb7b1
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/discovery/TimelineMetricMetadataManager.java
@@ -0,0 +1,606 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.metrics.core.timeline.discovery;
+
+import java.net.MalformedURLException;
+import java.net.URISyntaxException;
+import java.sql.SQLException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguration;
+import org.apache.ambari.metrics.core.timeline.uuid.MetricUuidGenStrategy;
+import org.apache.ambari.metrics.core.timeline.uuid.RandomUuidGenStrategy;
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.collections.MapUtils;
+import org.apache.commons.lang.ArrayUtils;
+import org.apache.commons.lang.StringUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.metrics2.sink.timeline.MetadataException;
+import org.apache.hadoop.metrics2.sink.timeline.TimelineMetric;
+import org.apache.hadoop.metrics2.sink.timeline.TimelineMetricMetadata;
+import org.apache.ambari.metrics.core.timeline.PhoenixHBaseAccessor;
+import org.apache.ambari.metrics.core.timeline.aggregators.TimelineClusterMetric;
+import org.apache.ambari.metrics.core.timeline.uuid.HashBasedUuidGenStrategy;
+
+public class TimelineMetricMetadataManager {
+ private static final Log LOG = LogFactory.getLog(TimelineMetricMetadataManager.class);
+ private boolean isDisabled = false;
+ // Cache all metadata on retrieval
+ private final Map<TimelineMetricMetadataKey, TimelineMetricMetadata> METADATA_CACHE = new ConcurrentHashMap<>();
+ private final Map<String, TimelineMetricMetadataKey> uuidKeyMap = new ConcurrentHashMap<>();
+ // Map to lookup apps on a host
+ private final Map<String, TimelineMetricHostMetadata> HOSTED_APPS_MAP = new ConcurrentHashMap<>();
+ private final Map<String, String> uuidHostMap = new ConcurrentHashMap<>();
+ private final Map<String, Set<String>> INSTANCE_HOST_MAP = new ConcurrentHashMap<>();
+ // Sync only when needed
+ AtomicBoolean SYNC_HOSTED_APPS_METADATA = new AtomicBoolean(false);
+ AtomicBoolean SYNC_HOSTED_INSTANCES_METADATA = new AtomicBoolean(false);
+ private MetricUuidGenStrategy uuidGenStrategy = new HashBasedUuidGenStrategy();
+ public static final int TIMELINE_METRIC_UUID_LENGTH = 16;
+ public static final int HOSTNAME_UUID_LENGTH = 4;
+
+ // Single thread to sync back new writes to the store
+ private final ScheduledExecutorService executorService = Executors.newSingleThreadScheduledExecutor();
+
+ private PhoenixHBaseAccessor hBaseAccessor;
+ private Configuration metricsConf;
+
+ TimelineMetricMetadataSync metricMetadataSync;
+ // Filter metrics names matching given patterns, from metadata
+ final List<String> metricNameFilters = new ArrayList<>();
+
+ // Test friendly construction since mock instrumentation is difficult to get
+ // working with hadoop mini cluster
+ public TimelineMetricMetadataManager(Configuration metricsConf, PhoenixHBaseAccessor hBaseAccessor) {
+ this.metricsConf = metricsConf;
+ this.hBaseAccessor = hBaseAccessor;
+ String patternStrings = metricsConf.get(TimelineMetricConfiguration.TIMELINE_METRIC_METADATA_FILTERS);
+ if (!StringUtils.isEmpty(patternStrings)) {
+ metricNameFilters.addAll(Arrays.asList(patternStrings.split(",")));
+ }
+
+ uuidGenStrategy = getUuidStrategy(metricsConf);
+ }
+
+ public TimelineMetricMetadataManager(PhoenixHBaseAccessor hBaseAccessor) throws MalformedURLException, URISyntaxException {
+ this(TimelineMetricConfiguration.getInstance().getMetricsConf(), hBaseAccessor);
+ }
+
+ /**
+ * Initialize Metadata from the store
+ */
+ public void initializeMetadata() {
+ if (metricsConf.getBoolean(TimelineMetricConfiguration.DISABLE_METRIC_METADATA_MGMT, false)) {
+ isDisabled = true;
+ } else {
+ metricMetadataSync = new TimelineMetricMetadataSync(this);
+ // Schedule the executor to sync to store
+ executorService.scheduleWithFixedDelay(metricMetadataSync,
+ metricsConf.getInt(TimelineMetricConfiguration.METRICS_METADATA_SYNC_INIT_DELAY, 120), // 2 minutes
+ metricsConf.getInt(TimelineMetricConfiguration.METRICS_METADATA_SYNC_SCHEDULE_DELAY, 300), // 5 minutes
+ TimeUnit.SECONDS);
+ // Read from store and initialize map
+ try {
+ Map<TimelineMetricMetadataKey, TimelineMetricMetadata> metadata = getMetadataFromStore();
+
+ LOG.info("Retrieved " + metadata.size() + ", metadata objects from store.");
+ // Store in the cache
+ METADATA_CACHE.putAll(metadata);
+
+ Map<String, TimelineMetricHostMetadata> hostedAppData = getHostedAppsFromStore();
+
+ LOG.info("Retrieved " + hostedAppData.size() + " host objects from store.");
+ HOSTED_APPS_MAP.putAll(hostedAppData);
+
+ loadUuidMapsOnInit();
+
+ hBaseAccessor.setMetadataInstance(this);
+ } catch (SQLException e) {
+ LOG.warn("Exception loading metric metadata", e);
+ }
+ }
+ }
+
+ public Map<TimelineMetricMetadataKey, TimelineMetricMetadata> getMetadataCache() {
+ return METADATA_CACHE;
+ }
+
+ public TimelineMetricMetadata getMetadataCacheValue(TimelineMetricMetadataKey key) {
+ return METADATA_CACHE.get(key);
+ }
+
+ public Map<String, TimelineMetricHostMetadata> getHostedAppsCache() {
+ return HOSTED_APPS_MAP;
+ }
+
+ public Map<String, Set<String>> getHostedInstanceCache() {
+ return INSTANCE_HOST_MAP;
+ }
+
+ public boolean syncHostedAppsMetadata() {
+ return SYNC_HOSTED_APPS_METADATA.get();
+ }
+
+ public boolean syncHostedInstanceMetadata() {
+ return SYNC_HOSTED_INSTANCES_METADATA.get();
+ }
+
+ public void markSuccessOnSyncHostedAppsMetadata() {
+ SYNC_HOSTED_APPS_METADATA.set(false);
+ }
+
+ public void markSuccessOnSyncHostedInstanceMetadata() {
+ SYNC_HOSTED_INSTANCES_METADATA.set(false);
+ }
+ /**
+ * Test metric name for valid patterns and return true/false
+ */
+ boolean skipMetadataCache(String metricName) {
+ for (String pattern : metricNameFilters) {
+ if (metricName.contains(pattern)) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ /**
+ * Update value in metadata cache
+ * @param metadata @TimelineMetricMetadata
+ */
+ public void putIfModifiedTimelineMetricMetadata(TimelineMetricMetadata metadata) {
+ if (skipMetadataCache(metadata.getMetricName())) {
+ return;
+ }
+
+ TimelineMetricMetadataKey key = new TimelineMetricMetadataKey(
+ metadata.getMetricName(), metadata.getAppId(), metadata.getInstanceId());
+
+ TimelineMetricMetadata metadataFromCache = METADATA_CACHE.get(key);
+
+ if (metadataFromCache != null) {
+ try {
+ if (metadataFromCache.needsToBeSynced(metadata)) {
+ metadata.setIsPersisted(false); // Set the flag to ensure sync to store on next run
+ METADATA_CACHE.put(key, metadata);
+ }
+ } catch (MetadataException e) {
+ LOG.warn("Error inserting Metadata in cache.", e);
+ }
+
+ } else {
+ METADATA_CACHE.put(key, metadata);
+ }
+ }
+
+ /**
+ * Update value in hosted apps cache
+ * @param hostname Host name
+ * @param appId Application Id
+ */
+ public void putIfModifiedHostedAppsMetadata(String hostname, String appId) {
+ TimelineMetricHostMetadata timelineMetricHostMetadata = HOSTED_APPS_MAP.get(hostname);
+ ConcurrentHashMap<String, String> apps = (timelineMetricHostMetadata != null) ? timelineMetricHostMetadata.getHostedApps() : null;
+ if (apps == null) {
+ apps = new ConcurrentHashMap<>();
+ if (timelineMetricHostMetadata == null) {
+ HOSTED_APPS_MAP.put(hostname, new TimelineMetricHostMetadata(apps));
+ } else {
+ HOSTED_APPS_MAP.get(hostname).setHostedApps(apps);
+ }
+ }
+
+ if (!apps.containsKey(appId)) {
+ apps.put(appId, appId);
+ SYNC_HOSTED_APPS_METADATA.set(true);
+ }
+ }
+
+ public void putIfModifiedHostedInstanceMetadata(String instanceId, String hostname) {
+ if (StringUtils.isEmpty(instanceId)) {
+ return;
+ }
+
+ Set<String> hosts = INSTANCE_HOST_MAP.get(instanceId);
+ if (hosts == null) {
+ hosts = new HashSet<>();
+ INSTANCE_HOST_MAP.put(instanceId, hosts);
+ }
+
+ if (!hosts.contains(hostname)) {
+ hosts.add(hostname);
+ SYNC_HOSTED_INSTANCES_METADATA.set(true);
+ }
+ }
+
+ public void persistMetadata(Collection<TimelineMetricMetadata> metadata) throws SQLException {
+ hBaseAccessor.saveMetricMetadata(metadata);
+ }
+
+ public void persistHostedAppsMetadata(Map<String, TimelineMetricHostMetadata> hostedApps) throws SQLException {
+ hBaseAccessor.saveHostAppsMetadata(hostedApps);
+ }
+
+ public void persistHostedInstanceMetadata(Map<String, Set<String>> hostedInstancesMetadata) throws SQLException {
+ hBaseAccessor.saveInstanceHostsMetadata(hostedInstancesMetadata);
+ }
+
+ public TimelineMetricMetadata getTimelineMetricMetadata(TimelineMetric timelineMetric, boolean isWhitelisted) {
+ return new TimelineMetricMetadata(
+ timelineMetric.getMetricName(),
+ timelineMetric.getAppId(),
+ timelineMetric.getInstanceId(),
+ timelineMetric.getUnits(),
+ timelineMetric.getType(),
+ timelineMetric.getStartTime(),
+ supportAggregates(timelineMetric),
+ isWhitelisted
+ );
+ }
+
+ public boolean isDisabled() {
+ return isDisabled;
+ }
+
+ boolean isDistributedModeEnabled() {
+ return metricsConf.get("timeline.metrics.service.operation.mode").equals("distributed");
+ }
+
+ /**
+ * Fetch metrics metadata from store
+ * @throws SQLException
+ */
+ Map<TimelineMetricMetadataKey, TimelineMetricMetadata> getMetadataFromStore() throws SQLException {
+ return hBaseAccessor.getTimelineMetricMetadata();
+ }
+
+ /**
+ * Fetch hosted apps from store
+ * @throws SQLException
+ */
+ Map<String, TimelineMetricHostMetadata> getHostedAppsFromStore() throws SQLException {
+ return hBaseAccessor.getHostedAppsMetadata();
+ }
+
+ Map<String, Set<String>> getHostedInstancesFromStore() throws SQLException {
+ return hBaseAccessor.getInstanceHostsMetdata();
+ }
+
+ private boolean supportAggregates(TimelineMetric metric) {
+ return MapUtils.isEmpty(metric.getMetadata()) ||
+ !(String.valueOf(true).equals(metric.getMetadata().get("skipAggregation")));
+ }
+
+ //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+ // UUID Management
+ //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+
+ /**
+ * Load the UUID mappings from the UUID table on startup.
+ */
+ private void loadUuidMapsOnInit() {
+
+ for (TimelineMetricMetadataKey key : METADATA_CACHE.keySet()) {
+ TimelineMetricMetadata timelineMetricMetadata = METADATA_CACHE.get(key);
+ if (timelineMetricMetadata != null && timelineMetricMetadata.getUuid() != null) {
+ uuidKeyMap.put(new String(timelineMetricMetadata.getUuid()), key);
+ }
+ }
+
+ for (String host : HOSTED_APPS_MAP.keySet()) {
+ TimelineMetricHostMetadata timelineMetricHostMetadata = HOSTED_APPS_MAP.get(host);
+ if (timelineMetricHostMetadata != null && timelineMetricHostMetadata.getUuid() != null) {
+ uuidHostMap.put(new String(timelineMetricHostMetadata.getUuid()), host);
+ }
+ }
+ }
+
+ /**
+ * Returns the UUID gen strategy.
+ * @param configuration
+ * @return
+ */
+ private MetricUuidGenStrategy getUuidStrategy(Configuration configuration) {
+ String strategy = configuration.get(TimelineMetricConfiguration.TIMELINE_METRICS_UUID_GEN_STRATEGY, "");
+ if ("random".equalsIgnoreCase(strategy)) {
+ return new RandomUuidGenStrategy();
+ } else {
+ return new HashBasedUuidGenStrategy();
+ }
+ }
+
+ /**
+ * Given the hostname, generates a byte array of length 'HOSTNAME_UUID_LENGTH'
+ * @param hostname
+ * @return uuid byte array of length 'HOSTNAME_UUID_LENGTH'
+ */
+ private byte[] getUuidForHostname(String hostname) {
+
+ TimelineMetricHostMetadata timelineMetricHostMetadata = HOSTED_APPS_MAP.get(hostname);
+ if (timelineMetricHostMetadata != null) {
+ byte[] uuid = timelineMetricHostMetadata.getUuid();
+ if (uuid != null) {
+ return uuid;
+ }
+ }
+
+ byte[] uuid = uuidGenStrategy.computeUuid(hostname, HOSTNAME_UUID_LENGTH);
+
+ String uuidStr = new String(uuid);
+ if (uuidHostMap.containsKey(uuidStr)) {
+ //TODO fix the collisions
+ LOG.error("Duplicate key computed for " + hostname +", Collides with " + uuidHostMap.get(uuidStr));
+ return uuid;
+ }
+
+ if (timelineMetricHostMetadata == null) {
+ timelineMetricHostMetadata = new TimelineMetricHostMetadata();
+ HOSTED_APPS_MAP.put(hostname, timelineMetricHostMetadata);
+ }
+ timelineMetricHostMetadata.setUuid(uuid);
+ uuidHostMap.put(uuidStr, hostname);
+
+ return uuid;
+ }
+
+ /**
+ * Given a timelineClusterMetric instance, generates a UUID for Metric-App-Instance combination.
+ * @param timelineClusterMetric
+ * @return uuid byte array of length 'TIMELINE_METRIC_UUID_LENGTH'
+ */
+ public byte[] getUuid(TimelineClusterMetric timelineClusterMetric) {
+ TimelineMetricMetadataKey key = new TimelineMetricMetadataKey(timelineClusterMetric.getMetricName(),
+ timelineClusterMetric.getAppId(), timelineClusterMetric.getInstanceId());
+
+ TimelineMetricMetadata timelineMetricMetadata = METADATA_CACHE.get(key);
+ if (timelineMetricMetadata != null) {
+ byte[] uuid = timelineMetricMetadata.getUuid();
+ if (uuid != null) {
+ return uuid;
+ }
+ }
+
+ byte[] uuid = uuidGenStrategy.computeUuid(timelineClusterMetric, TIMELINE_METRIC_UUID_LENGTH);
+
+ String uuidStr = new String(uuid);
+ if (uuidKeyMap.containsKey(uuidStr) && !uuidKeyMap.get(uuidStr).equals(key)) {
+ TimelineMetricMetadataKey collidingKey = (TimelineMetricMetadataKey)uuidKeyMap.get(uuidStr);
+ //TODO fix the collisions
+ /**
+ * 2017-08-23 14:12:35,922 ERROR TimelineMetricMetadataManager:
+ * Duplicate key [52, 50, 51, 53, 50, 53, 53, 53, 49, 54, 57, 50, 50, 54, 0, 0]([B@278a93f9) computed for
+ * TimelineClusterMetric{metricName='sdisk_dm-11_write_count', appId='hbase', instanceId='', timestamp=1503497400000}, Collides with
+ * TimelineMetricMetadataKey{metricName='sdisk_dm-20_write_count', appId='hbase', instanceId=''}
+ */
+ LOG.error("Duplicate key " + Arrays.toString(uuid) + "(" + uuid + ") computed for " + timelineClusterMetric.toString() + ", Collides with " + collidingKey.toString());
+ return uuid;
+ }
+
+ if (timelineMetricMetadata == null) {
+ timelineMetricMetadata = new TimelineMetricMetadata();
+ timelineMetricMetadata.setMetricName(timelineClusterMetric.getMetricName());
+ timelineMetricMetadata.setAppId(timelineClusterMetric.getAppId());
+ timelineMetricMetadata.setInstanceId(timelineClusterMetric.getInstanceId());
+ METADATA_CACHE.put(key, timelineMetricMetadata);
+ }
+
+ timelineMetricMetadata.setUuid(uuid);
+ timelineMetricMetadata.setIsPersisted(false);
+ uuidKeyMap.put(uuidStr, key);
+ return uuid;
+ }
+
+ /**
+ * Given a timelineMetric instance, generates a UUID for Metric-App-Instance combination.
+ * @param timelineMetric
+ * @return uuid byte array of length 'TIMELINE_METRIC_UUID_LENGTH' + 'HOSTNAME_UUID_LENGTH'
+ */
+ public byte[] getUuid(TimelineMetric timelineMetric) {
+
+ byte[] metricUuid = getUuid(new TimelineClusterMetric(timelineMetric.getMetricName(), timelineMetric.getAppId(),
+ timelineMetric.getInstanceId(), -1l));
+ byte[] hostUuid = getUuidForHostname(timelineMetric.getHostName());
+
+ return ArrayUtils.addAll(metricUuid, hostUuid);
+ }
+
+ public byte[] getUuid(String metricName, String appId, String instanceId, String hostname) {
+
+ byte[] metricUuid = getUuid(new TimelineClusterMetric(metricName, appId, instanceId, -1l));
+ if (StringUtils.isNotEmpty(hostname)) {
+ byte[] hostUuid = getUuidForHostname(hostname);
+ return ArrayUtils.addAll(metricUuid, hostUuid);
+ }
+ return metricUuid;
+ }
+
+ public String getMetricNameFromUuid(byte[] uuid) {
+
+ byte[] metricUuid = uuid;
+ if (uuid.length == TIMELINE_METRIC_UUID_LENGTH + HOSTNAME_UUID_LENGTH) {
+ metricUuid = ArrayUtils.subarray(uuid, 0, TIMELINE_METRIC_UUID_LENGTH);
+ }
+
+ TimelineMetricMetadataKey key = uuidKeyMap.get(new String(metricUuid));
+ return key != null ? key.getMetricName() : null;
+ }
+
+ public TimelineMetric getMetricFromUuid(byte[] uuid) {
+ if (uuid == null) {
+ return null;
+ }
+
+ if (uuid.length == TIMELINE_METRIC_UUID_LENGTH) {
+ TimelineMetricMetadataKey key = uuidKeyMap.get(new String(uuid));
+ return key != null ? new TimelineMetric(key.metricName, null, key.appId, key.instanceId) : null;
+ } else {
+ byte[] metricUuid = ArrayUtils.subarray(uuid, 0, TIMELINE_METRIC_UUID_LENGTH);
+ TimelineMetricMetadataKey key = uuidKeyMap.get(new String(metricUuid));
+ if (key == null) {
+ LOG.error("TimelineMetricMetadataKey is null for : " + Arrays.toString(uuid));
+ return null;
+ }
+ TimelineMetric timelineMetric = new TimelineMetric();
+ timelineMetric.setMetricName(key.metricName);
+ timelineMetric.setAppId(key.appId);
+ timelineMetric.setInstanceId(key.instanceId);
+
+ byte[] hostUuid = ArrayUtils.subarray(uuid, TIMELINE_METRIC_UUID_LENGTH, HOSTNAME_UUID_LENGTH + TIMELINE_METRIC_UUID_LENGTH);
+ timelineMetric.setHostName(uuidHostMap.get(new String(hostUuid)));
+ return timelineMetric;
+ }
+ }
+
+ /**
+ * Returns the set of UUIDs for a given GET request. If there are wildcards (%), resolves them based on UUID map.
+ * @param metricNames
+ * @param hostnames
+ * @param appId
+ * @param instanceId
+ * @return Set of UUIds
+ */
+ public List<byte[]> getUuids(Collection<String> metricNames, List<String> hostnames, String appId, String instanceId) {
+
+ Collection<String> sanitizedMetricNames = new HashSet<>();
+
+ for (String metricName : metricNames) {
+ if (metricName.contains("%")) {
+ String metricRegEx;
+ //Special case handling for metric name with * and __%.
+ //For example, dfs.NNTopUserOpCounts.windowMs=300000.op=*.user=%.count
+ // or dfs.NNTopUserOpCounts.windowMs=300000.op=__%.user=%.count
+ if (metricName.contains("*") || metricName.contains("__%")) {
+ String metricNameWithEscSeq = metricName.replace("*", "\\*").replace("__%", "..%");
+ metricRegEx = metricNameWithEscSeq.replace("%", ".*");
+ } else {
+ metricRegEx = metricName.replace("%", ".*");
+ }
+ for (TimelineMetricMetadataKey key : METADATA_CACHE.keySet()) {
+ String metricNameFromMetadata = key.getMetricName();
+ if (metricNameFromMetadata.matches(metricRegEx)) {
+ sanitizedMetricNames.add(metricNameFromMetadata);
+ }
+ }
+ } else {
+ sanitizedMetricNames.add(metricName);
+ }
+ }
+
+ Set<String> sanitizedHostNames = new HashSet<>();
+ if (CollectionUtils.isNotEmpty(hostnames)) {
+ for (String hostname : hostnames) {
+ if (hostname.contains("%")) {
+ String hostRegEx;
+ hostRegEx = hostname.replace("%", ".*");
+ for (String host : HOSTED_APPS_MAP.keySet()) {
+ if (host.matches(hostRegEx)) {
+ sanitizedHostNames.add(host);
+ }
+ }
+ } else {
+ sanitizedHostNames.add(hostname);
+ }
+ }
+ }
+
+ List<byte[]> uuids = new ArrayList<>();
+
+ if ( StringUtils.isNotEmpty(appId) && !(appId.equals("HOST") || appId.equals("FLUME_HANDLER"))) { //HACK.. Why??
+ appId = appId.toLowerCase();
+ }
+ if (CollectionUtils.isNotEmpty(sanitizedHostNames)) {
+ if (CollectionUtils.isNotEmpty(sanitizedMetricNames)) {
+ for (String metricName : sanitizedMetricNames) {
+ TimelineMetric metric = new TimelineMetric();
+ metric.setMetricName(metricName);
+ metric.setAppId(appId);
+ metric.setInstanceId(instanceId);
+ for (String hostname : sanitizedHostNames) {
+ metric.setHostName(hostname);
+ byte[] uuid = getUuid(metric);
+ if (uuid != null) {
+ uuids.add(uuid);
+ }
+ }
+ }
+ } else {
+ for (String hostname : sanitizedHostNames) {
+ byte[] uuid = getUuidForHostname(hostname);
+ if (uuid != null) {
+ uuids.add(uuid);
+ }
+ }
+ }
+ } else {
+ for (String metricName : sanitizedMetricNames) {
+ TimelineClusterMetric metric = new TimelineClusterMetric(metricName, appId, instanceId, -1l);
+ byte[] uuid = getUuid(metric);
+ if (uuid != null) {
+ uuids.add(uuid);
+ }
+ }
+ }
+
+ return uuids;
+ }
+
+ public Map<String, TimelineMetricMetadataKey> getUuidKeyMap() {
+ return uuidKeyMap;
+ }
+
+ public List<String> getNotLikeHostnames(List<String> hostnames) {
+ List<String> result = new ArrayList<>();
+ Set<String> sanitizedHostNames = new HashSet<>();
+ if (CollectionUtils.isNotEmpty(hostnames)) {
+ for (String hostname : hostnames) {
+ if (hostname.contains("%")) {
+ String hostRegEx;
+ hostRegEx = hostname.replace("%", ".*");
+ for (String host : HOSTED_APPS_MAP.keySet()) {
+ if (host.matches(hostRegEx)) {
+ sanitizedHostNames.add(host);
+ }
+ }
+ } else {
+ sanitizedHostNames.add(hostname);
+ }
+ }
+ }
+
+ for (String hostname: HOSTED_APPS_MAP.keySet()) {
+ if (!sanitizedHostNames.contains(hostname)) {
+ result.add(hostname);
+ }
+ }
+ return result;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/discovery/TimelineMetricMetadataSync.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/discovery/TimelineMetricMetadataSync.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/discovery/TimelineMetricMetadataSync.java
new file mode 100644
index 0000000..0ffff76
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/discovery/TimelineMetricMetadataSync.java
@@ -0,0 +1,227 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.metrics.core.timeline.discovery;
+
+import java.sql.SQLException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.metrics2.sink.timeline.TimelineMetricMetadata;
+
+/**
+ * Sync metadata info with the store
+ */
+public class TimelineMetricMetadataSync implements Runnable {
+ private static final Log LOG = LogFactory.getLog(TimelineMetricMetadataSync.class);
+
+ private final TimelineMetricMetadataManager cacheManager;
+
+ public TimelineMetricMetadataSync(TimelineMetricMetadataManager cacheManager) {
+ this.cacheManager = cacheManager;
+ }
+
+ @Override
+ public void run() {
+ LOG.debug("Persisting metric metadata...");
+ persistMetricMetadata();
+ LOG.debug("Persisting hosted apps metadata...");
+ persistHostAppsMetadata();
+ LOG.debug("Persisting hosted instance metadata...");
+ persistHostInstancesMetadata();
+ if (cacheManager.isDistributedModeEnabled()) {
+ LOG.debug("Refreshing metric metadata...");
+ refreshMetricMetadata();
+ LOG.debug("Refreshing hosted apps metadata...");
+ refreshHostAppsMetadata();
+ LOG.debug("Refreshing hosted instances metadata...");
+ refreshHostedInstancesMetadata();
+ }
+ }
+
+ /**
+ * Find metrics not persisted to store and persist them
+ */
+ private void persistMetricMetadata() {
+ List<TimelineMetricMetadata> metadataToPersist = new ArrayList<>();
+ // Find all entries to persist
+ for (TimelineMetricMetadata metadata : cacheManager.getMetadataCache().values()) {
+ if (!metadata.isPersisted()) {
+ metadataToPersist.add(metadata);
+ }
+ }
+ boolean markSuccess = false;
+ if (!metadataToPersist.isEmpty()) {
+ try {
+ cacheManager.persistMetadata(metadataToPersist);
+ markSuccess = true;
+ } catch (SQLException e) {
+ LOG.warn("Error persisting metadata.", e);
+ }
+ }
+ // Mark corresponding entries as persisted to skip on next run
+ if (markSuccess) {
+ for (TimelineMetricMetadata metadata : metadataToPersist) {
+ TimelineMetricMetadataKey key = new TimelineMetricMetadataKey(
+ metadata.getMetricName(), metadata.getAppId(), metadata.getInstanceId()
+ );
+
+ // Mark entry as being persisted
+ metadata.setIsPersisted(true);
+ // Update cache
+ cacheManager.getMetadataCache().put(key, metadata);
+ }
+ }
+ }
+
+ /**
+ * Read all metric metadata and update cached values - HA mode
+ */
+ private void refreshMetricMetadata() {
+ Map<TimelineMetricMetadataKey, TimelineMetricMetadata> metadataFromStore = null;
+ try {
+ metadataFromStore = cacheManager.getMetadataFromStore();
+ } catch (SQLException e) {
+ LOG.warn("Error refreshing metadata from store.", e);
+ }
+ if (metadataFromStore != null) {
+ Map<TimelineMetricMetadataKey, TimelineMetricMetadata> cachedMetadata =
+ cacheManager.getMetadataCache();
+
+ for (Map.Entry<TimelineMetricMetadataKey, TimelineMetricMetadata> metadataEntry : metadataFromStore.entrySet()) {
+ if (!cachedMetadata.containsKey(metadataEntry.getKey())) {
+ cachedMetadata.put(metadataEntry.getKey(), metadataEntry.getValue());
+ }
+ }
+ }
+ }
+
+ /**
+ * Sync hosted apps data if needed
+ */
+ private void persistHostAppsMetadata() {
+ if (cacheManager.syncHostedAppsMetadata()) {
+ Map<String, TimelineMetricHostMetadata> persistedData = null;
+ try {
+ persistedData = cacheManager.getHostedAppsFromStore();
+ } catch (SQLException e) {
+ LOG.warn("Failed on fetching hosted apps data from store.", e);
+ return; // Something wrong with store
+ }
+
+ Map<String, TimelineMetricHostMetadata> cachedData = cacheManager.getHostedAppsCache();
+ Map<String, TimelineMetricHostMetadata> dataToSync = new HashMap<>();
+ if (cachedData != null && !cachedData.isEmpty()) {
+ for (Map.Entry<String, TimelineMetricHostMetadata> cacheEntry : cachedData.entrySet()) {
+ // No persistence / stale data in store
+ if (persistedData == null || persistedData.isEmpty() ||
+ !persistedData.containsKey(cacheEntry.getKey()) ||
+ !persistedData.get(cacheEntry.getKey()).getHostedApps().keySet().containsAll(cacheEntry.getValue().getHostedApps().keySet())) {
+ dataToSync.put(cacheEntry.getKey(), cacheEntry.getValue());
+ }
+ }
+ try {
+ cacheManager.persistHostedAppsMetadata(dataToSync);
+ cacheManager.markSuccessOnSyncHostedAppsMetadata();
+
+ } catch (SQLException e) {
+ LOG.warn("Error persisting hosted apps metadata.", e);
+ }
+ }
+
+ }
+ }
+
+ /**
+ * Sync apps instances data if needed
+ */
+ private void persistHostInstancesMetadata() {
+ if (cacheManager.syncHostedInstanceMetadata()) {
+ Map<String, Set<String>> persistedData = null;
+ try {
+ persistedData = cacheManager.getHostedInstancesFromStore();
+ } catch (SQLException e) {
+ LOG.warn("Failed on fetching hosted instances data from store.", e);
+ return; // Something wrong with store
+ }
+
+ Map<String, Set<String>> cachedData = cacheManager.getHostedInstanceCache();
+ Map<String, Set<String>> dataToSync = new HashMap<>();
+ if (cachedData != null && !cachedData.isEmpty()) {
+ for (Map.Entry<String, Set<String>> cacheEntry : cachedData.entrySet()) {
+ // No persistence / stale data in store
+ if (persistedData == null || persistedData.isEmpty() ||
+ !persistedData.containsKey(cacheEntry.getKey()) ||
+ !persistedData.get(cacheEntry.getKey()).containsAll(cacheEntry.getValue())) {
+ dataToSync.put(cacheEntry.getKey(), cacheEntry.getValue());
+ }
+ }
+ try {
+ cacheManager.persistHostedInstanceMetadata(dataToSync);
+ cacheManager.markSuccessOnSyncHostedInstanceMetadata();
+
+ } catch (SQLException e) {
+ LOG.warn("Error persisting hosted apps metadata.", e);
+ }
+ }
+
+ }
+ }
+ /**
+ * Read all hosted apps metadata and update cached values - HA
+ */
+ private void refreshHostAppsMetadata() {
+ Map<String, TimelineMetricHostMetadata> hostedAppsDataFromStore = null;
+ try {
+ hostedAppsDataFromStore = cacheManager.getHostedAppsFromStore();
+ } catch (SQLException e) {
+ LOG.warn("Error refreshing metadata from store.", e);
+ }
+ if (hostedAppsDataFromStore != null) {
+ Map<String, TimelineMetricHostMetadata> cachedData = cacheManager.getHostedAppsCache();
+
+ for (Map.Entry<String, TimelineMetricHostMetadata> storeEntry : hostedAppsDataFromStore.entrySet()) {
+ if (!cachedData.containsKey(storeEntry.getKey())) {
+ cachedData.put(storeEntry.getKey(), storeEntry.getValue());
+ }
+ }
+ }
+ }
+
+ private void refreshHostedInstancesMetadata() {
+ Map<String, Set<String>> hostedInstancesFromStore = null;
+ try {
+ hostedInstancesFromStore = cacheManager.getHostedInstancesFromStore();
+ } catch (SQLException e) {
+ LOG.warn("Error refreshing metadata from store.", e);
+ }
+ if (hostedInstancesFromStore != null) {
+ Map<String, Set<String>> cachedData = cacheManager.getHostedInstanceCache();
+
+ for (Map.Entry<String, Set<String>> storeEntry : hostedInstancesFromStore.entrySet()) {
+ if (!cachedData.containsKey(storeEntry.getKey())) {
+ cachedData.put(storeEntry.getKey(), storeEntry.getValue());
+ }
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/function/AbstractTimelineMetricsSeriesAggregateFunction.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/function/AbstractTimelineMetricsSeriesAggregateFunction.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/function/AbstractTimelineMetricsSeriesAggregateFunction.java
new file mode 100644
index 0000000..1070242
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/function/AbstractTimelineMetricsSeriesAggregateFunction.java
@@ -0,0 +1,98 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.metrics.core.timeline.function;
+
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeMap;
+import java.util.TreeSet;
+
+import org.apache.hadoop.metrics2.sink.timeline.TimelineMetric;
+import org.apache.hadoop.metrics2.sink.timeline.TimelineMetrics;
+
+import com.google.common.base.Joiner;
+
+public abstract class AbstractTimelineMetricsSeriesAggregateFunction
+ implements TimelineMetricsSeriesAggregateFunction {
+
+ @Override
+ public TimelineMetric apply(TimelineMetrics timelineMetrics) {
+ Set<String> metricNameSet = new TreeSet<>();
+ Set<String> hostNameSet = new TreeSet<>();
+ Set<String> appIdSet = new TreeSet<>();
+ Set<String> instanceIdSet = new TreeSet<>();
+ TreeMap<Long, List<Double>> metricValues = new TreeMap<>();
+
+ for (TimelineMetric timelineMetric : timelineMetrics.getMetrics()) {
+ metricNameSet.add(timelineMetric.getMetricName());
+ addToSetOnlyNotNull(hostNameSet, timelineMetric.getHostName());
+ addToSetOnlyNotNull(appIdSet, timelineMetric.getAppId());
+ addToSetOnlyNotNull(instanceIdSet, timelineMetric.getInstanceId());
+
+ for (Map.Entry<Long, Double> metricValue : timelineMetric.getMetricValues().entrySet()) {
+ Long timestamp = metricValue.getKey();
+ Double value = metricValue.getValue();
+ if (!metricValues.containsKey(timestamp)) {
+ metricValues.put(timestamp, new LinkedList<Double>());
+ }
+ metricValues.get(timestamp).add(value);
+ }
+ }
+
+ TreeMap<Long, Double> aggregatedMetricValues = new TreeMap<>();
+ for (Map.Entry<Long, List<Double>> metricValue : metricValues.entrySet()) {
+ List<Double> values = metricValue.getValue();
+ if (values.size() == 0) {
+ throw new IllegalArgumentException("count of values should be more than 0");
+ }
+ aggregatedMetricValues.put(metricValue.getKey(), applyFunction(values));
+ }
+
+ TimelineMetric timelineMetric = new TimelineMetric();
+ timelineMetric.setMetricName(getMetricName(metricNameSet.iterator()));
+ timelineMetric.setHostName(joinStringsWithComma(hostNameSet.iterator()));
+ timelineMetric.setAppId(joinStringsWithComma(appIdSet.iterator()));
+ timelineMetric.setInstanceId(joinStringsWithComma(instanceIdSet.iterator()));
+ if (aggregatedMetricValues.size() > 0) {
+ timelineMetric.setStartTime(aggregatedMetricValues.firstKey());
+ }
+ timelineMetric.setMetricValues(aggregatedMetricValues);
+ return timelineMetric;
+ }
+
+ protected String getMetricName(Iterator<String> metricNames) {
+ return getFunctionName() + "(" + Joiner.on(",").join(metricNames) + ")";
+ }
+
+ protected String joinStringsWithComma(Iterator<String> hostNames) {
+ return Joiner.on(",").join(hostNames);
+ }
+
+ protected abstract Double applyFunction(List<Double> values);
+ protected abstract String getFunctionName();
+
+ private void addToSetOnlyNotNull(Set<String> set, String value) {
+ if (value != null) {
+ set.add(value);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/function/SeriesAggregateFunction.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/function/SeriesAggregateFunction.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/function/SeriesAggregateFunction.java
new file mode 100644
index 0000000..1438194
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/function/SeriesAggregateFunction.java
@@ -0,0 +1,42 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.metrics.core.timeline.function;
+
+import org.apache.ambari.metrics.core.timeline.aggregators.Function;
+
+public enum SeriesAggregateFunction {
+ AVG, MIN, MAX, SUM;
+
+ public static boolean isPresent(String functionName) {
+ try {
+ SeriesAggregateFunction.valueOf(functionName.toUpperCase());
+ } catch (IllegalArgumentException e) {
+ return false;
+ }
+ return true;
+ }
+
+ public static SeriesAggregateFunction getFunction(String functionName) throws Function.FunctionFormatException {
+ try {
+ return SeriesAggregateFunction.valueOf(functionName.toUpperCase());
+ } catch (NullPointerException | IllegalArgumentException e) {
+ throw new Function.FunctionFormatException(
+ "Function should be sum, avg, min, max. Got " + functionName, e);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/function/TimelineMetricsSeriesAggregateFunction.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/function/TimelineMetricsSeriesAggregateFunction.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/function/TimelineMetricsSeriesAggregateFunction.java
new file mode 100644
index 0000000..a5f40e1
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/function/TimelineMetricsSeriesAggregateFunction.java
@@ -0,0 +1,25 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.metrics.core.timeline.function;
+
+import org.apache.hadoop.metrics2.sink.timeline.TimelineMetric;
+import org.apache.hadoop.metrics2.sink.timeline.TimelineMetrics;
+
+public interface TimelineMetricsSeriesAggregateFunction {
+ TimelineMetric apply(TimelineMetrics timelineMetrics);
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/function/TimelineMetricsSeriesAggregateFunctionFactory.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/function/TimelineMetricsSeriesAggregateFunctionFactory.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/function/TimelineMetricsSeriesAggregateFunctionFactory.java
new file mode 100644
index 0000000..f07aeb9
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/function/TimelineMetricsSeriesAggregateFunctionFactory.java
@@ -0,0 +1,41 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.metrics.core.timeline.function;
+
+import org.apache.ambari.metrics.core.timeline.aggregators.Function;
+
+public class TimelineMetricsSeriesAggregateFunctionFactory {
+ private TimelineMetricsSeriesAggregateFunctionFactory() {
+ }
+
+ public static TimelineMetricsSeriesAggregateFunction newInstance(SeriesAggregateFunction func) {
+ switch (func) {
+ case AVG:
+ return new TimelineMetricsSeriesAvgAggregateFunction();
+ case MIN:
+ return new TimelineMetricsSeriesMinAggregateFunction();
+ case MAX:
+ return new TimelineMetricsSeriesMaxAggregateFunction();
+ case SUM:
+ return new TimelineMetricsSeriesSumAggregateFunction();
+ default:
+ throw new Function.FunctionFormatException("Function should be sum, avg, min, max. Got " +
+ func.name());
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/function/TimelineMetricsSeriesAvgAggregateFunction.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/function/TimelineMetricsSeriesAvgAggregateFunction.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/function/TimelineMetricsSeriesAvgAggregateFunction.java
new file mode 100644
index 0000000..b0e3069
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/function/TimelineMetricsSeriesAvgAggregateFunction.java
@@ -0,0 +1,39 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.metrics.core.timeline.function;
+
+import java.util.List;
+
+public class TimelineMetricsSeriesAvgAggregateFunction extends AbstractTimelineMetricsSeriesAggregateFunction {
+ private static final String FUNCTION_NAME = "AVG";
+
+ @Override
+ protected Double applyFunction(List<Double> values) {
+ double sum = 0.0d;
+ for (Double value : values) {
+ sum += value;
+ }
+
+ return sum / values.size();
+ }
+
+ @Override
+ protected String getFunctionName() {
+ return FUNCTION_NAME;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/function/TimelineMetricsSeriesMaxAggregateFunction.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/function/TimelineMetricsSeriesMaxAggregateFunction.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/function/TimelineMetricsSeriesMaxAggregateFunction.java
new file mode 100644
index 0000000..58c6d4c
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/function/TimelineMetricsSeriesMaxAggregateFunction.java
@@ -0,0 +1,41 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.metrics.core.timeline.function;
+
+import java.util.List;
+
+public class TimelineMetricsSeriesMaxAggregateFunction extends AbstractTimelineMetricsSeriesAggregateFunction {
+ private static final String FUNCTION_NAME = "MAX";
+
+ @Override
+ protected Double applyFunction(List<Double> values) {
+ double max = Double.MIN_VALUE;
+ for (Double value : values) {
+ if (value > max) {
+ max = value;
+ }
+ }
+
+ return max;
+ }
+
+ @Override
+ protected String getFunctionName() {
+ return FUNCTION_NAME;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/function/TimelineMetricsSeriesMinAggregateFunction.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/function/TimelineMetricsSeriesMinAggregateFunction.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/function/TimelineMetricsSeriesMinAggregateFunction.java
new file mode 100644
index 0000000..60e1bb5
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/function/TimelineMetricsSeriesMinAggregateFunction.java
@@ -0,0 +1,41 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.metrics.core.timeline.function;
+
+import java.util.List;
+
+public class TimelineMetricsSeriesMinAggregateFunction extends AbstractTimelineMetricsSeriesAggregateFunction {
+ private static final String FUNCTION_NAME = "MIN";
+
+ @Override
+ protected Double applyFunction(List<Double> values) {
+ double min = Double.MAX_VALUE;
+ for (Double value : values) {
+ if (value < min) {
+ min = value;
+ }
+ }
+
+ return min;
+ }
+
+ @Override
+ protected String getFunctionName() {
+ return FUNCTION_NAME;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/function/TimelineMetricsSeriesSumAggregateFunction.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/function/TimelineMetricsSeriesSumAggregateFunction.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/function/TimelineMetricsSeriesSumAggregateFunction.java
new file mode 100644
index 0000000..997d7be
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/function/TimelineMetricsSeriesSumAggregateFunction.java
@@ -0,0 +1,39 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.metrics.core.timeline.function;
+
+import java.util.List;
+
+public class TimelineMetricsSeriesSumAggregateFunction extends AbstractTimelineMetricsSeriesAggregateFunction {
+ private static final String FUNCTION_NAME = "SUM";
+
+ @Override
+ protected Double applyFunction(List<Double> values) {
+ double sum = 0.0d;
+ for (Double value : values) {
+ sum += value;
+ }
+
+ return sum;
+ }
+
+ @Override
+ protected String getFunctionName() {
+ return FUNCTION_NAME;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/query/Condition.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/query/Condition.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/query/Condition.java
new file mode 100644
index 0000000..fa118a3
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/query/Condition.java
@@ -0,0 +1,51 @@
+package org.apache.ambari.metrics.core.timeline.query;
+
+import java.util.List;
+
+import org.apache.hadoop.metrics2.sink.timeline.Precision;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+public interface Condition {
+ boolean isEmpty();
+
+ List<byte[]> getUuids();
+ List<String> getMetricNames();
+ boolean isPointInTime();
+ boolean isGrouped();
+ void setStatement(String statement);
+ List<String> getHostnames();
+ Precision getPrecision();
+ void setPrecision(Precision precision);
+ String getAppId();
+ String getInstanceId();
+ StringBuilder getConditionClause();
+ String getOrderByClause(boolean asc);
+ String getStatement();
+ Long getStartTime();
+ Long getEndTime();
+ Integer getLimit();
+ Integer getFetchSize();
+ void setFetchSize(Integer fetchSize);
+ void addOrderByColumn(String column);
+ void setNoLimit();
+ boolean doUpdate();
+ void setMetricNamesNotCondition(boolean metricNamesNotCondition);
+ void setHostnamesNotCondition(boolean hostNamesNotCondition);
+ void setUuidNotCondition(boolean uuidNotCondition);
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/query/ConditionBuilder.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/query/ConditionBuilder.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/query/ConditionBuilder.java
new file mode 100644
index 0000000..e779d77
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/query/ConditionBuilder.java
@@ -0,0 +1,143 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.metrics.core.timeline.query;
+
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.hadoop.metrics2.sink.timeline.Precision;
+import org.apache.ambari.metrics.core.timeline.aggregators.Function;
+
+public class ConditionBuilder {
+
+ private List<String> metricNames;
+ private List<String> hostnames;
+ private String appId;
+ private String instanceId;
+ private Long startTime;
+ private Long endTime;
+ private Precision precision;
+ private Integer limit;
+ private boolean grouped;
+ private boolean noLimit = false;
+ private Integer fetchSize;
+ private String statement;
+ private Set<String> orderByColumns = new LinkedHashSet<String>();
+ private Integer topN;
+ private boolean isBottomN;
+ private Function topNFunction;
+ private List<byte[]> uuids;
+
+ public ConditionBuilder(List<String> metricNames) {
+ this.metricNames = metricNames;
+ }
+
+ public ConditionBuilder hostnames(List<String> hostnames) {
+ this.hostnames = hostnames;
+ return this;
+ }
+
+ public ConditionBuilder appId(String appId) {
+ this.appId = appId;
+ return this;
+ }
+
+ public ConditionBuilder instanceId(String instanceId) {
+ this.instanceId = instanceId;
+ return this;
+ }
+
+ public ConditionBuilder startTime(Long startTime) {
+ this.startTime = startTime;
+ return this;
+ }
+
+ public ConditionBuilder endTime(Long endTime) {
+ this.endTime = endTime;
+ return this;
+ }
+
+ public ConditionBuilder precision(Precision precision) {
+ this.precision = precision;
+ return this;
+ }
+
+ public ConditionBuilder limit(Integer limit) {
+ this.limit = limit;
+ return this;
+ }
+
+ public ConditionBuilder grouped(boolean grouped) {
+ this.grouped = grouped;
+ return this;
+ }
+
+ public ConditionBuilder noLimit(boolean noLimit) {
+ this.noLimit = noLimit;
+ return this;
+ }
+
+ public ConditionBuilder fetchSize(Integer fetchSize) {
+ this.fetchSize = fetchSize;
+ return this;
+ }
+
+ public ConditionBuilder statement(String statement) {
+ this.statement = statement;
+ return this;
+ }
+
+ public ConditionBuilder orderByColumns(Set<String> orderByColumns) {
+ this.orderByColumns = orderByColumns;
+ return this;
+ }
+
+ public ConditionBuilder topN(Integer topN) {
+ this.topN = topN;
+ return this;
+ }
+
+ public ConditionBuilder isBottomN(boolean isBottomN) {
+ this.isBottomN = isBottomN;
+ return this;
+ }
+
+ public ConditionBuilder topNFunction(Function topNFunction) {
+ this.topNFunction = topNFunction;
+ return this;
+ }
+
+ public ConditionBuilder uuid(List<byte[]> uuids) {
+ this.uuids = uuids;
+ return this;
+ }
+
+ public Condition build() {
+ if (topN == null) {
+ return new DefaultCondition(
+ uuids, metricNames,
+ hostnames, appId, instanceId, startTime, endTime,
+ precision, limit, grouped);
+ } else {
+ return new TopNCondition(uuids, metricNames, hostnames, appId, instanceId,
+ startTime, endTime, precision, limit, grouped, topN, topNFunction, isBottomN);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/query/ConnectionProvider.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/query/ConnectionProvider.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/query/ConnectionProvider.java
new file mode 100644
index 0000000..830a0eb
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/query/ConnectionProvider.java
@@ -0,0 +1,29 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.metrics.core.timeline.query;
+
+
+import java.sql.Connection;
+import java.sql.SQLException;
+
+/**
+ *
+ */
+public interface ConnectionProvider {
+ public Connection getConnection() throws SQLException;
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/query/DefaultCondition.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/query/DefaultCondition.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/query/DefaultCondition.java
new file mode 100644
index 0000000..888f381
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/query/DefaultCondition.java
@@ -0,0 +1,421 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.metrics.core.timeline.query;
+
+import java.util.ArrayList;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.metrics2.sink.timeline.Precision;
+import org.apache.ambari.metrics.core.timeline.PhoenixHBaseAccessor;
+import org.apache.ambari.metrics.core.timeline.discovery.TimelineMetricMetadataManager;
+
+public class DefaultCondition implements Condition {
+ List<String> metricNames;
+ List<String> hostnames;
+ String appId;
+ String instanceId;
+ Long startTime;
+ Long endTime;
+ Precision precision;
+ Integer limit;
+ boolean grouped;
+ boolean noLimit = false;
+ Integer fetchSize;
+ String statement;
+ Set<String> orderByColumns = new LinkedHashSet<String>();
+ boolean metricNamesNotCondition = false;
+ boolean hostNamesNotCondition = false;
+ boolean uuidNotCondition = false;
+ List<byte[]> uuids = new ArrayList<>();
+
+ private static final Log LOG = LogFactory.getLog(DefaultCondition.class);
+
+ public DefaultCondition(List<String> metricNames, List<String> hostnames, String appId,
+ String instanceId, Long startTime, Long endTime, Precision precision,
+ Integer limit, boolean grouped) {
+ this.metricNames = metricNames;
+ this.hostnames = hostnames;
+ this.appId = appId;
+ this.instanceId = instanceId;
+ this.startTime = startTime;
+ this.endTime = endTime;
+ this.precision = precision;
+ this.limit = limit;
+ this.grouped = grouped;
+ }
+
+ public DefaultCondition(List<byte[]> uuids, List<String> metricNames, List<String> hostnames, String appId,
+ String instanceId, Long startTime, Long endTime, Precision precision,
+ Integer limit, boolean grouped) {
+ this.uuids = uuids;
+ this.metricNames = metricNames;
+ this.hostnames = hostnames;
+ this.appId = appId;
+ this.instanceId = instanceId;
+ this.startTime = startTime;
+ this.endTime = endTime;
+ this.precision = precision;
+ this.limit = limit;
+ this.grouped = grouped;
+ }
+
+ public String getStatement() {
+ return statement;
+ }
+
+ public void setStatement(String statement) {
+ this.statement = statement;
+ }
+
+ public List<String> getMetricNames() {
+ return metricNames == null || metricNames.isEmpty() ? null : metricNames;
+ }
+
+ public StringBuilder getConditionClause() {
+ StringBuilder sb = new StringBuilder();
+ boolean appendConjunction = appendUuidClause(sb);
+ appendConjunction = append(sb, appendConjunction, getStartTime(), " SERVER_TIME >= ?");
+ append(sb, appendConjunction, getEndTime(), " SERVER_TIME < ?");
+
+ return sb;
+ }
+
+ protected static boolean append(StringBuilder sb,
+ boolean appendConjunction,
+ Object value, String str) {
+ if (value != null) {
+ if (appendConjunction) {
+ sb.append(" AND");
+ }
+
+ sb.append(str);
+ appendConjunction = true;
+ }
+ return appendConjunction;
+ }
+
+ public List<String> getHostnames() {
+ return hostnames;
+ }
+
+ public Precision getPrecision() {
+ return precision;
+ }
+
+ public void setPrecision(Precision precision) {
+ this.precision = precision;
+ }
+
+ public String getAppId() {
+ if (appId != null && !appId.isEmpty()) {
+ if (!(appId.equals("HOST") || appId.equals("FLUME_HANDLER"))) {
+ return appId.toLowerCase();
+ } else {
+ return appId;
+ }
+ }
+ return null;
+ }
+
+ public String getInstanceId() {
+ return instanceId == null || instanceId.isEmpty() ? null : instanceId;
+ }
+
+ /**
+ * Convert to millis.
+ */
+ public Long getStartTime() {
+ if (startTime == null) {
+ return null;
+ } else if (startTime < 9999999999l) {
+ return startTime * 1000;
+ } else {
+ return startTime;
+ }
+ }
+
+ public Long getEndTime() {
+ if (endTime == null) {
+ return null;
+ }
+ if (endTime < 9999999999l) {
+ return endTime * 1000;
+ } else {
+ return endTime;
+ }
+ }
+
+ public void setNoLimit() {
+ this.noLimit = true;
+ }
+
+ @Override
+ public boolean doUpdate() {
+ return false;
+ }
+
+ public Integer getLimit() {
+ if (noLimit) {
+ return null;
+ }
+ return limit == null ? PhoenixHBaseAccessor.RESULTSET_LIMIT : limit;
+ }
+
+ public boolean isGrouped() {
+ return grouped;
+ }
+
+ public boolean isPointInTime() {
+ return getStartTime() == null && getEndTime() == null;
+ }
+
+ public boolean isEmpty() {
+ return (metricNames == null || metricNames.isEmpty())
+ && (hostnames == null || hostnames.isEmpty())
+ && (appId == null || appId.isEmpty())
+ && (instanceId == null || instanceId.isEmpty())
+ && startTime == null
+ && endTime == null;
+ }
+
+ public Integer getFetchSize() {
+ return fetchSize;
+ }
+
+ public void setFetchSize(Integer fetchSize) {
+ this.fetchSize = fetchSize;
+ }
+
+ public void addOrderByColumn(String column) {
+ orderByColumns.add(column);
+ }
+
+ public String getOrderByClause(boolean asc) {
+ String orderByStr = " ORDER BY ";
+ if (!orderByColumns.isEmpty()) {
+ StringBuilder sb = new StringBuilder(orderByStr);
+ for (String orderByColumn : orderByColumns) {
+ if (sb.length() != orderByStr.length()) {
+ sb.append(", ");
+ }
+ sb.append(orderByColumn);
+ if (!asc) {
+ sb.append(" DESC");
+ }
+ }
+ sb.append(" ");
+ return sb.toString();
+ }
+ return null;
+ }
+
+ protected boolean appendUuidClause(StringBuilder sb) {
+ boolean appendConjunction = false;
+
+ if (CollectionUtils.isNotEmpty(uuids)) {
+
+ List<byte[]> uuidsHost = new ArrayList<>();
+ List<byte[]> uuidsMetric = new ArrayList<>();
+ List<byte[]> uuidsFull = new ArrayList<>();
+
+ if (getUuids() != null) {
+ for (byte[] uuid : uuids) {
+ if (uuid.length == TimelineMetricMetadataManager.TIMELINE_METRIC_UUID_LENGTH) {
+ uuidsMetric.add(uuid);
+ } else if (uuid.length == TimelineMetricMetadataManager.HOSTNAME_UUID_LENGTH) {
+ uuidsHost.add(uuid);
+ } else {
+ uuidsFull.add(uuid);
+ }
+ }
+
+ // Put a '(' first
+ sb.append("(");
+
+ //IN clause
+ // METRIC_NAME (NOT) IN (?,?,?,?)
+ if (CollectionUtils.isNotEmpty(uuidsFull)) {
+ sb.append("UUID");
+ if (uuidNotCondition) {
+ sb.append(" NOT");
+ }
+ sb.append(" IN (");
+ //Append ?,?,?,?
+ for (int i = 0; i < uuidsFull.size(); i++) {
+ sb.append("?");
+ if (i < uuidsFull.size() - 1) {
+ sb.append(", ");
+ }
+ }
+ sb.append(")");
+ appendConjunction = true;
+ }
+
+ //Put an AND if both types are present
+ if (CollectionUtils.isNotEmpty(uuidsFull) &&
+ CollectionUtils.isNotEmpty(uuidsMetric)) {
+ sb.append(" AND ");
+ }
+
+ // ( for OR
+ if (!metricNamesNotCondition && uuidsMetric.size() > 1 && (CollectionUtils.isNotEmpty(uuidsFull) || CollectionUtils.isNotEmpty(uuidsHost))) {
+ sb.append("(");
+ }
+
+ //LIKE clause for clusterMetric UUIDs
+ // UUID (NOT) LIKE ? OR(AND) UUID LIKE ?
+ if (CollectionUtils.isNotEmpty(uuidsMetric)) {
+
+ for (int i = 0; i < uuidsMetric.size(); i++) {
+ sb.append("UUID");
+ if (metricNamesNotCondition) {
+ sb.append(" NOT");
+ }
+ sb.append(" LIKE ");
+ sb.append("?");
+
+ if (i < uuidsMetric.size() - 1) {
+ if (metricNamesNotCondition) {
+ sb.append(" AND ");
+ } else {
+ sb.append(" OR ");
+ }
+ // ) for OR
+ } else if ((CollectionUtils.isNotEmpty(uuidsFull) || CollectionUtils.isNotEmpty(uuidsHost)) && !metricNamesNotCondition && uuidsMetric.size() > 1) {
+ sb.append(")");
+ }
+ }
+ appendConjunction = true;
+ }
+
+ //Put an AND if both types are present
+ if ((CollectionUtils.isNotEmpty(uuidsMetric) || (CollectionUtils.isNotEmpty(uuidsFull) && CollectionUtils.isEmpty(uuidsMetric)))
+ && CollectionUtils.isNotEmpty(uuidsHost)) {
+ sb.append(" AND ");
+ }
+ // ( for OR
+ if((CollectionUtils.isNotEmpty(uuidsFull) || CollectionUtils.isNotEmpty(uuidsMetric)) && !hostNamesNotCondition && uuidsHost.size() > 1){
+ sb.append("(");
+ }
+
+ //LIKE clause for HOST UUIDs
+ // UUID (NOT) LIKE ? OR(AND) UUID LIKE ?
+ if (CollectionUtils.isNotEmpty(uuidsHost)) {
+
+ for (int i = 0; i < uuidsHost.size(); i++) {
+ sb.append("UUID");
+ if (hostNamesNotCondition) {
+ sb.append(" NOT");
+ }
+ sb.append(" LIKE ");
+ sb.append("?");
+
+ if (i < uuidsHost.size() - 1) {
+ if (hostNamesNotCondition) {
+ sb.append(" AND ");
+ } else {
+ sb.append(" OR ");
+ }
+ // ) for OR
+ } else if ((CollectionUtils.isNotEmpty(uuidsFull) || CollectionUtils.isNotEmpty(uuidsMetric)) && !hostNamesNotCondition && uuidsHost.size() > 1) {
+ sb.append(")");
+ }
+ }
+ appendConjunction = true;
+ }
+
+ // Finish with a ')'
+ if (appendConjunction) {
+ sb.append(")");
+ }
+
+ uuids = new ArrayList<>();
+ if (CollectionUtils.isNotEmpty(uuidsFull)) {
+ uuids.addAll(uuidsFull);
+ }
+ for (byte[] uuid: uuidsMetric) {
+ uuids.add(new String(uuid).concat("%").getBytes());
+ }
+ for (byte[] uuid: uuidsHost) {
+ uuids.add("%".concat(new String(uuid)).getBytes());
+ }
+ }
+ }
+
+ return appendConjunction;
+ }
+
+ @Override
+ public String toString() {
+ return "Condition{" +
+ "uuids=" + uuids +
+ ", appId='" + appId + '\'' +
+ ", instanceId='" + instanceId + '\'' +
+ ", startTime=" + startTime +
+ ", endTime=" + endTime +
+ ", limit=" + limit +
+ ", grouped=" + grouped +
+ ", orderBy=" + orderByColumns +
+ ", noLimit=" + noLimit +
+ '}';
+ }
+
+ protected static boolean metricNamesHaveWildcard(List<String> metricNames) {
+ for (String name : metricNames) {
+ if (name.contains("%")) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ protected static boolean hostNamesHaveWildcard(List<String> hostnames) {
+ if (hostnames == null)
+ return false;
+ for (String name : hostnames) {
+ if (name.contains("%")) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ public void setMetricNamesNotCondition(boolean metricNamesNotCondition) {
+ this.metricNamesNotCondition = metricNamesNotCondition;
+ }
+
+ @Override
+ public void setHostnamesNotCondition(boolean hostNamesNotCondition) {
+ this.hostNamesNotCondition = hostNamesNotCondition;
+ }
+
+ @Override
+ public void setUuidNotCondition(boolean uuidNotCondition) {
+ this.uuidNotCondition = uuidNotCondition;
+ }
+
+ @Override
+ public List<byte[]> getUuids() {
+ return uuids;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/query/DefaultPhoenixDataSource.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/query/DefaultPhoenixDataSource.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/query/DefaultPhoenixDataSource.java
new file mode 100644
index 0000000..5c0a4b5
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/query/DefaultPhoenixDataSource.java
@@ -0,0 +1,90 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.metrics.core.timeline.query;
+
+
+import java.io.IOException;
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.SQLException;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.client.ConnectionFactory;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
+
+public class DefaultPhoenixDataSource implements PhoenixConnectionProvider {
+
+ static final Log LOG = LogFactory.getLog(DefaultPhoenixDataSource.class);
+ private static final String ZOOKEEPER_CLIENT_PORT = "hbase.zookeeper.property.clientPort";
+ private static final String ZOOKEEPER_QUORUM = "hbase.zookeeper.quorum";
+ private static final String ZNODE_PARENT = "zookeeper.znode.parent";
+
+ private static final String connectionUrl = "jdbc:phoenix:%s:%s:%s";
+ private final String url;
+
+ private Configuration hbaseConf;
+
+ public DefaultPhoenixDataSource(Configuration hbaseConf) {
+ this.hbaseConf = hbaseConf;
+ String zookeeperClientPort = hbaseConf.getTrimmed(ZOOKEEPER_CLIENT_PORT, "2181");
+ String zookeeperQuorum = hbaseConf.getTrimmed(ZOOKEEPER_QUORUM);
+ String znodeParent = hbaseConf.getTrimmed(ZNODE_PARENT, "/ams-hbase-unsecure");
+ if (zookeeperQuorum == null || zookeeperQuorum.isEmpty()) {
+ throw new IllegalStateException("Unable to find Zookeeper quorum to " +
+ "access HBase store using Phoenix.");
+ }
+
+ url = String.format(connectionUrl,
+ zookeeperQuorum,
+ zookeeperClientPort,
+ znodeParent);
+ }
+
+ /**
+ * Get HBaseAdmin for table ops.
+ * @return @HBaseAdmin
+ * @throws IOException
+ */
+ public HBaseAdmin getHBaseAdmin() throws IOException {
+ return (HBaseAdmin) ConnectionFactory.createConnection(hbaseConf).getAdmin();
+ }
+
+ /**
+ * Get JDBC connection to HBase store. Assumption is that the hbase
+ * configuration is present on the classpath and loaded by the caller into
+ * the Configuration object.
+ * Phoenix already caches the HConnection between the client and HBase
+ * cluster.
+ *
+ * @return @java.sql.Connection
+ */
+ public Connection getConnection() throws SQLException {
+
+ LOG.debug("Metric store connection url: " + url);
+ try {
+ return DriverManager.getConnection(url);
+ } catch (SQLException e) {
+ LOG.warn("Unable to connect to HBase store using Phoenix.", e);
+
+ throw e;
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/query/EmptyCondition.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/query/EmptyCondition.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/query/EmptyCondition.java
new file mode 100644
index 0000000..742b09b
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/query/EmptyCondition.java
@@ -0,0 +1,169 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.metrics.core.timeline.query;
+
+import java.util.List;
+
+import org.apache.commons.lang.NotImplementedException;
+import org.apache.hadoop.metrics2.sink.timeline.Precision;
+
+/**
+ * Encapsulate a Condition with pre-formatted and pre-parsed query string.
+ */
+public class EmptyCondition implements Condition {
+ String statement;
+ boolean doUpdate = false;
+ boolean metricNamesNotCondition = false;
+
+ @Override
+ public boolean isEmpty() {
+ return false;
+ }
+
+ @Override
+ public List<byte[]> getUuids() {
+ return null;
+ }
+
+ @Override
+ public List<String> getMetricNames() {
+ return null;
+ }
+
+ @Override
+ public boolean isPointInTime() {
+ return false;
+ }
+
+ @Override
+ public boolean isGrouped() {
+ return true;
+ }
+
+ @Override
+ public void setStatement(String statement) {
+ this.statement = statement;
+ }
+
+ @Override
+ public List<String> getHostnames() {
+ return null;
+ }
+
+ @Override
+ public Precision getPrecision() {
+ return null;
+ }
+
+ @Override
+ public void setPrecision(Precision precision) {
+
+ }
+
+ @Override
+ public String getAppId() {
+ return null;
+ }
+
+ @Override
+ public String getInstanceId() {
+ return null;
+ }
+
+ @Override
+ public StringBuilder getConditionClause() {
+ return null;
+ }
+
+ @Override
+ public String getOrderByClause(boolean asc) {
+ return null;
+ }
+
+ @Override
+ public String getStatement() {
+ return statement;
+ }
+
+ @Override
+ public Long getStartTime() {
+ return null;
+ }
+
+ @Override
+ public Long getEndTime() {
+ return null;
+ }
+
+ @Override
+ public Integer getLimit() {
+ return null;
+ }
+
+ @Override
+ public Integer getFetchSize() {
+ return null;
+ }
+
+ @Override
+ public void setFetchSize(Integer fetchSize) {
+
+ }
+
+ @Override
+ public void addOrderByColumn(String column) {
+
+ }
+
+ @Override
+ public void setNoLimit() {
+
+ }
+
+ public void setDoUpdate(boolean doUpdate) {
+ this.doUpdate = doUpdate;
+ }
+
+ @Override
+ public boolean doUpdate() {
+ return doUpdate;
+ }
+
+ @Override
+ public String toString() {
+ return "EmptyCondition{ " +
+ " statement = " + this.getStatement() +
+ " doUpdate = " + this.doUpdate() +
+ " }";
+ }
+
+ @Override
+ public void setMetricNamesNotCondition(boolean metricNamesNotCondition) {
+ this.metricNamesNotCondition = metricNamesNotCondition;
+ }
+
+ @Override
+ public void setHostnamesNotCondition(boolean hostNamesNotCondition) {
+ throw new NotImplementedException("Not implemented");
+ }
+
+ @Override
+ public void setUuidNotCondition(boolean uuidNotCondition) {
+ throw new NotImplementedException("Not implemented");
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/query/PhoenixConnectionProvider.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/query/PhoenixConnectionProvider.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/query/PhoenixConnectionProvider.java
new file mode 100644
index 0000000..c8f958e
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/query/PhoenixConnectionProvider.java
@@ -0,0 +1,31 @@
+package org.apache.ambari.metrics.core.timeline.query;
+
+import java.io.IOException;
+
+import org.apache.hadoop.hbase.client.Admin;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+public interface PhoenixConnectionProvider extends ConnectionProvider {
+ /**
+ * Get HBaseAdmin for the Phoenix connection
+ * @return
+ * @throws IOException
+ */
+ Admin getHBaseAdmin() throws IOException;
+}
[07/28] ambari git commit: AMBARI-22740 : Fix integration test for
HBase in branch-3.0-ams due to UUID changes. (avijayan)
Posted by av...@apache.org.
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/timeline/TimelineMetricStoreWatcherTest.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/timeline/TimelineMetricStoreWatcherTest.java b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/timeline/TimelineMetricStoreWatcherTest.java
new file mode 100644
index 0000000..de0236c
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/timeline/TimelineMetricStoreWatcherTest.java
@@ -0,0 +1,108 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.metrics.core.timeline;
+
+import org.apache.hadoop.metrics2.sink.timeline.Precision;
+import org.apache.hadoop.metrics2.sink.timeline.TimelineMetrics;
+import org.apache.hadoop.metrics2.sink.timeline.TopNConfig;
+import org.apache.hadoop.util.ExitUtil;
+import org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse;
+import org.easymock.EasyMock;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.powermock.core.classloader.annotations.PrepareForTest;
+import org.powermock.modules.junit4.PowerMockRunner;
+
+import java.util.List;
+
+import static org.easymock.EasyMock.anyObject;
+import static org.easymock.EasyMock.anyString;
+import static org.easymock.EasyMock.createNiceMock;
+import static org.easymock.EasyMock.eq;
+import static org.easymock.EasyMock.expect;
+import static org.easymock.EasyMock.expectLastCall;
+import static org.easymock.EasyMock.replay;
+import static org.easymock.EasyMock.verify;
+import static org.powermock.api.easymock.PowerMock.mockStatic;
+import static org.powermock.api.easymock.PowerMock.replayAll;
+import static org.powermock.api.easymock.PowerMock.verifyAll;
+
+@RunWith(PowerMockRunner.class)
+@PrepareForTest(ExitUtil.class)
+public class TimelineMetricStoreWatcherTest {
+
+ @Test
+ public void testRunPositive() throws Exception {
+ TimelineMetricStore metricStore = createNiceMock(TimelineMetricStore.class);
+
+ expect(metricStore.putMetrics(anyObject(TimelineMetrics.class)))
+ .andReturn(new TimelinePutResponse());
+
+ // metric found
+ expect(metricStore.getTimelineMetrics(EasyMock.<List<String>>anyObject(),
+ EasyMock.<List<String>>anyObject(), anyObject(String.class),
+ anyObject(String.class), anyObject(Long.class), anyObject(Long.class),
+ eq(Precision.SECONDS), eq(1), eq(true), anyObject(TopNConfig.class), anyString()))
+ .andReturn(null).anyTimes();
+
+ mockStatic(ExitUtil.class);
+
+ replay(metricStore);
+
+ TimelineMetricStoreWatcher timelineMetricStoreWatcher =
+ new TimelineMetricStoreWatcher(metricStore, TimelineMetricConfiguration.getInstance());
+ timelineMetricStoreWatcher.run();
+ timelineMetricStoreWatcher.run();
+ timelineMetricStoreWatcher.run();
+ verify(metricStore);
+
+ }
+
+ @Test
+ public void testRunNegative() throws Exception {
+ TimelineMetricStore metricStore = createNiceMock(TimelineMetricStore.class);
+
+ expect(metricStore.putMetrics(anyObject(TimelineMetrics.class)))
+ .andReturn(new TimelinePutResponse());
+
+ // no metrics found
+ expect(metricStore.getTimelineMetrics(EasyMock.<List<String>>anyObject(),
+ EasyMock.<List<String>>anyObject(), anyObject(String.class),
+ anyObject(String.class), anyObject(Long.class), anyObject(Long.class),
+ eq(Precision.SECONDS), eq(1), eq(true), anyObject(TopNConfig.class), anyString()))
+ .andReturn(null).anyTimes();
+
+ String msg = "Error getting metrics from TimelineMetricStore. " +
+ "Shutting down by TimelineMetricStoreWatcher.";
+ mockStatic(ExitUtil.class);
+ ExitUtil.terminate(-1, msg);
+ expectLastCall().anyTimes();
+
+ replayAll();
+
+ TimelineMetricStoreWatcher timelineMetricStoreWatcher =
+ new TimelineMetricStoreWatcher(metricStore, TimelineMetricConfiguration.getInstance());
+ timelineMetricStoreWatcher.run();
+ timelineMetricStoreWatcher.run();
+ timelineMetricStoreWatcher.run();
+
+ verifyAll();
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/timeline/TimelineMetricsAggregatorMemorySink.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/timeline/TimelineMetricsAggregatorMemorySink.java b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/timeline/TimelineMetricsAggregatorMemorySink.java
new file mode 100644
index 0000000..11a01d9
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/timeline/TimelineMetricsAggregatorMemorySink.java
@@ -0,0 +1,141 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.metrics.core.timeline;
+
+import org.apache.hadoop.metrics2.sink.timeline.MetricClusterAggregate;
+import org.apache.hadoop.metrics2.sink.timeline.MetricHostAggregate;
+import org.apache.hadoop.metrics2.sink.timeline.Precision;
+import org.apache.hadoop.metrics2.sink.timeline.TimelineMetric;
+import org.apache.ambari.metrics.core.timeline.aggregators.TimelineClusterMetric;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Map.Entry;
+
+/**
+ * Aggregator Memory sink implementation to perform test
+ */
+public class TimelineMetricsAggregatorMemorySink
+ implements TimelineMetricsAggregatorSink {
+
+ private static Map<Precision, Map<TimelineMetric, MetricHostAggregate>> hostAggregateRecords =
+ new HashMap<>();
+ private static Map<Precision, Map<TimelineClusterMetric, MetricHostAggregate>> clusterTimeAggregateRecords =
+ new HashMap<>();
+ private static Map<TimelineClusterMetric, MetricClusterAggregate> clusterAggregateRecords =
+ new HashMap<>();
+
+ @Override
+ public void saveHostAggregateRecords(
+ Map<TimelineMetric, MetricHostAggregate> hostAggregateMap,
+ Precision precision) {
+ if (hostAggregateMap == null || hostAggregateMap.size() == 0) {
+ return;
+ }
+
+ Map<TimelineMetric, MetricHostAggregate> aggregatedValue = null;
+ if (hostAggregateRecords.containsKey(precision)) {
+ aggregatedValue = hostAggregateRecords.get(precision);
+ } else {
+ aggregatedValue = new HashMap<>();
+ hostAggregateRecords.put(precision, aggregatedValue);
+ }
+
+ for (Entry<TimelineMetric, MetricHostAggregate> entry : hostAggregateMap
+ .entrySet()) {
+ TimelineMetric timelineMetricClone = new TimelineMetric(entry.getKey());
+ MetricHostAggregate hostAggregate = entry.getValue();
+ MetricHostAggregate hostAggregateClone = new MetricHostAggregate(
+ hostAggregate.getSum(), (int) hostAggregate.getNumberOfSamples(),
+ hostAggregate.getDeviation(), hostAggregate.getMax(),
+ hostAggregate.getMin());
+ aggregatedValue.put(timelineMetricClone, hostAggregateClone);
+ }
+ }
+
+ @Override
+ public void saveClusterTimeAggregateRecords(
+ Map<TimelineClusterMetric, MetricHostAggregate> clusterTimeAggregateMap,
+ Precision precision) {
+ if (clusterTimeAggregateMap == null
+ || clusterTimeAggregateMap.size() == 0) {
+ return;
+ }
+
+ Map<TimelineClusterMetric, MetricHostAggregate> aggregatedValue = null;
+ if (clusterTimeAggregateRecords.containsKey(precision)) {
+ aggregatedValue = clusterTimeAggregateRecords.get(precision);
+ } else {
+ aggregatedValue = new HashMap<>();
+ clusterTimeAggregateRecords.put(precision, aggregatedValue);
+ }
+
+ for (Entry<TimelineClusterMetric, MetricHostAggregate> entry : clusterTimeAggregateMap
+ .entrySet()) {
+ TimelineClusterMetric clusterMetric = entry.getKey();
+ TimelineClusterMetric clusterMetricClone =
+ new TimelineClusterMetric(clusterMetric.getMetricName(),
+ clusterMetric.getAppId(), clusterMetric.getInstanceId(),
+ clusterMetric.getTimestamp());
+ MetricHostAggregate hostAggregate = entry.getValue();
+ MetricHostAggregate hostAggregateClone = new MetricHostAggregate(
+ hostAggregate.getSum(), (int) hostAggregate.getNumberOfSamples(),
+ hostAggregate.getDeviation(), hostAggregate.getMax(),
+ hostAggregate.getMin());
+ aggregatedValue.put(clusterMetricClone, hostAggregateClone);
+ }
+ }
+
+ @Override
+ public void saveClusterAggregateRecords(
+ Map<TimelineClusterMetric, MetricClusterAggregate> clusterAggregateMaps) {
+
+ if (clusterAggregateMaps == null || clusterAggregateMaps.size() == 0) {
+ return;
+ }
+
+ for (Entry<TimelineClusterMetric, MetricClusterAggregate> entry : clusterAggregateMaps
+ .entrySet()) {
+ TimelineClusterMetric clusterMetric = entry.getKey();
+ TimelineClusterMetric clusterMetricClone =
+ new TimelineClusterMetric(clusterMetric.getMetricName(),
+ clusterMetric.getAppId(), clusterMetric.getInstanceId(),
+ clusterMetric.getTimestamp());
+ MetricClusterAggregate clusterAggregate = entry.getValue();
+ MetricClusterAggregate clusterAggregateClone = new MetricClusterAggregate(
+ clusterAggregate.getSum(), (int) clusterAggregate.getNumberOfHosts(),
+ clusterAggregate.getDeviation(), clusterAggregate.getMax(),
+ clusterAggregate.getMin());
+ clusterAggregateRecords.put(clusterMetricClone, clusterAggregateClone);
+ }
+ }
+
+ public Map<Precision, Map<TimelineMetric, MetricHostAggregate>> getHostAggregateRecords() {
+ return Collections.unmodifiableMap(hostAggregateRecords);
+ }
+
+ public Map<Precision, Map<TimelineClusterMetric, MetricHostAggregate>> getClusterTimeAggregateRecords() {
+ return Collections.unmodifiableMap(clusterTimeAggregateRecords);
+ }
+
+ public Map<TimelineClusterMetric, MetricClusterAggregate> getClusterAggregateRecords() {
+ return Collections.unmodifiableMap(clusterAggregateRecords);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/timeline/TimelineMetricsFilterTest.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/timeline/TimelineMetricsFilterTest.java b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/timeline/TimelineMetricsFilterTest.java
new file mode 100644
index 0000000..d64bf7c
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/timeline/TimelineMetricsFilterTest.java
@@ -0,0 +1,223 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.metrics.core.timeline;
+
+import static org.easymock.EasyMock.expect;
+import static org.easymock.EasyMock.replay;
+
+import junit.framework.Assert;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.metrics2.sink.timeline.TimelineMetric;
+import org.easymock.EasyMock;
+import org.junit.Test;
+
+import java.net.URL;
+import java.util.HashSet;
+import java.util.Set;
+
+public class TimelineMetricsFilterTest {
+
+ @Test
+ public void testAppBlacklisting() throws Exception{
+
+ Configuration metricsConf = new Configuration();
+ metricsConf.set("timeline.metrics.apps.blacklist", "hbase,datanode,nimbus");
+ TimelineMetricConfiguration configuration = EasyMock.createNiceMock(TimelineMetricConfiguration.class);
+ expect(configuration.getMetricsConf()).andReturn(metricsConf).once();
+ replay(configuration);
+
+ TimelineMetricsFilter.initializeMetricFilter(configuration);
+
+ TimelineMetric timelineMetric = new TimelineMetric();
+
+ timelineMetric.setAppId("hbase");
+ Assert.assertFalse(TimelineMetricsFilter.acceptMetric(timelineMetric));
+
+ timelineMetric.setAppId("namenode");
+ Assert.assertTrue(TimelineMetricsFilter.acceptMetric(timelineMetric));
+
+ timelineMetric.setAppId("nimbus");
+ Assert.assertFalse(TimelineMetricsFilter.acceptMetric(timelineMetric));
+ }
+
+ @Test
+ public void testMetricWhitelisting() throws Exception {
+
+ Configuration metricsConf = new Configuration();
+ TimelineMetricConfiguration configuration = EasyMock.createNiceMock(TimelineMetricConfiguration.class);
+ expect(configuration.getMetricsConf()).andReturn(metricsConf).once();
+ expect(configuration.isWhitelistingEnabled()).andReturn(true).anyTimes();
+ replay(configuration);
+
+ URL fileUrl = ClassLoader.getSystemResource("test_data/metric_whitelist.dat");
+
+ metricsConf.set("timeline.metrics.whitelist.file", fileUrl.getPath());
+ TimelineMetricsFilter.initializeMetricFilter(configuration);
+
+ TimelineMetric timelineMetric = new TimelineMetric();
+
+ timelineMetric.setMetricName("cpu_system");
+ Assert.assertTrue(TimelineMetricsFilter.acceptMetric(timelineMetric));
+
+ timelineMetric.setMetricName("cpu_system1");
+ Assert.assertFalse(TimelineMetricsFilter.acceptMetric(timelineMetric));
+
+ timelineMetric.setMetricName("jvm.JvmMetrics.MemHeapUsedM");
+ Assert.assertTrue(TimelineMetricsFilter.acceptMetric(timelineMetric));
+
+ timelineMetric.setMetricName("dfs.FSNamesystem.TotalFiles");
+ Assert.assertTrue(TimelineMetricsFilter.acceptMetric(timelineMetric));
+ }
+
+ @Test
+ public void testTogether() throws Exception {
+
+ Configuration metricsConf = new Configuration();
+ metricsConf.set("timeline.metrics.apps.blacklist", "hbase,datanode,nimbus");
+ TimelineMetricConfiguration configuration = EasyMock.createNiceMock(TimelineMetricConfiguration.class);
+ expect(configuration.getMetricsConf()).andReturn(metricsConf).once();
+ replay(configuration);
+
+ URL fileUrl = ClassLoader.getSystemResource("test_data/metric_whitelist.dat");
+ metricsConf.set("timeline.metrics.whitelist.file", fileUrl.getPath());
+
+ TimelineMetricsFilter.initializeMetricFilter(configuration);
+
+ TimelineMetric timelineMetric = new TimelineMetric();
+
+ timelineMetric.setMetricName("cpu_system");
+ timelineMetric.setAppId("hbase");
+ Assert.assertFalse(TimelineMetricsFilter.acceptMetric(timelineMetric));
+
+ timelineMetric.setMetricName("cpu_system");
+ timelineMetric.setAppId("HOST");
+ Assert.assertTrue(TimelineMetricsFilter.acceptMetric(timelineMetric));
+
+ timelineMetric.setMetricName("jvm.JvmMetrics.MemHeapUsedM");
+ Assert.assertTrue(TimelineMetricsFilter.acceptMetric(timelineMetric));
+
+ timelineMetric.setMetricName("dfs.FSNamesystem.TotalFiles");
+ Assert.assertTrue(TimelineMetricsFilter.acceptMetric(timelineMetric));
+ }
+
+ @Test
+ public void testAmshbaseWhitelisting() throws Exception {
+
+ TimelineMetricConfiguration configuration = EasyMock.createNiceMock(TimelineMetricConfiguration.class);
+
+ Configuration metricsConf = new Configuration();
+ expect(configuration.getMetricsConf()).andReturn(metricsConf).once();
+
+ Set<String> whitelist = new HashSet();
+ whitelist.add("regionserver.Server.Delete_99th_percentile");
+ whitelist.add("regionserver.Server.Delete_max");
+ whitelist.add("regionserver.Server.Delete_mean");
+ expect(configuration.getAmshbaseWhitelist()).andReturn(whitelist).once();
+
+ replay(configuration);
+
+ TimelineMetricsFilter.initializeMetricFilter(configuration);
+
+ TimelineMetric timelineMetric = new TimelineMetric();
+
+ timelineMetric.setMetricName("regionserver.Server.Delete_max");
+ timelineMetric.setAppId("ams-hbase");
+ Assert.assertTrue(TimelineMetricsFilter.acceptMetric(timelineMetric));
+
+ timelineMetric.setMetricName("regionserver.Server.Delete_min3333");
+ timelineMetric.setAppId("ams-hbase");
+ Assert.assertFalse(TimelineMetricsFilter.acceptMetric(timelineMetric));
+
+ timelineMetric.setMetricName("jvm.JvmMetrics.MemHeapUsedM");
+ timelineMetric.setAppId("hbase");
+ Assert.assertTrue(TimelineMetricsFilter.acceptMetric(timelineMetric));
+ }
+
+ @Test
+ public void testHybridFilter() throws Exception {
+
+ // Whitelist Apps - namenode, nimbus
+ // Blacklist Apps - datanode, kafka_broker
+ // Accept ams-hbase whitelisting.
+ // Reject non whitelisted metrics from non whitelisted Apps (Say hbase)
+
+ TimelineMetricConfiguration configuration = EasyMock.createNiceMock(TimelineMetricConfiguration.class);
+
+ Configuration metricsConf = new Configuration();
+ metricsConf.set("timeline.metrics.apps.whitelist", "namenode,nimbus");
+ metricsConf.set("timeline.metrics.apps.blacklist", "datanode,kafka_broker");
+ URL fileUrl = ClassLoader.getSystemResource("test_data/metric_whitelist.dat");
+ metricsConf.set("timeline.metrics.whitelist.file", fileUrl.getPath());
+ expect(configuration.getMetricsConf()).andReturn(metricsConf).once();
+
+ Set<String> whitelist = new HashSet();
+ whitelist.add("regionserver.Server.Delete_99th_percentile");
+ whitelist.add("regionserver.Server.Delete_max");
+ whitelist.add("regionserver.Server.Delete_mean");
+ expect(configuration.getAmshbaseWhitelist()).andReturn(whitelist).once();
+
+ expect(configuration.isWhitelistingEnabled()).andReturn(true).anyTimes();
+
+ replay(configuration);
+
+ TimelineMetricsFilter.initializeMetricFilter(configuration);
+
+ TimelineMetric timelineMetric = new TimelineMetric();
+
+ //Test App Whitelisting
+ timelineMetric.setMetricName("metric.a.b.c");
+ timelineMetric.setAppId("namenode");
+ Assert.assertTrue(TimelineMetricsFilter.acceptMetric(timelineMetric));
+
+ timelineMetric.setMetricName("metric.d.e.f");
+ timelineMetric.setAppId("nimbus");
+ Assert.assertTrue(TimelineMetricsFilter.acceptMetric(timelineMetric));
+
+ //Test App Blacklisting
+ timelineMetric.setMetricName("metric.d.e.f");
+ timelineMetric.setAppId("datanode");
+ Assert.assertFalse(TimelineMetricsFilter.acceptMetric(timelineMetric));
+
+ timelineMetric.setMetricName("metric.d.e.f");
+ timelineMetric.setAppId("kafka_broker");
+ Assert.assertFalse(TimelineMetricsFilter.acceptMetric(timelineMetric));
+
+
+ //Test ams-hbase Whitelisting
+ timelineMetric.setMetricName("regionserver.Server.Delete_max");
+ timelineMetric.setAppId("ams-hbase");
+ Assert.assertTrue(TimelineMetricsFilter.acceptMetric(timelineMetric));
+
+ timelineMetric.setMetricName("regionserver.Server.Delete_min3333");
+ timelineMetric.setAppId("ams-hbase");
+ Assert.assertFalse(TimelineMetricsFilter.acceptMetric(timelineMetric));
+
+ timelineMetric.setMetricName("regionserver.Server.Delete_mean");
+ timelineMetric.setAppId("ams-hbase");
+ Assert.assertTrue(TimelineMetricsFilter.acceptMetric(timelineMetric));
+
+ //Test Metric Whitelisting
+ timelineMetric.setMetricName("regionserver.WAL.SyncTime_max");
+ timelineMetric.setAppId("hbase");
+ Assert.assertTrue(TimelineMetricsFilter.acceptMetric(timelineMetric));
+
+ timelineMetric.setMetricName("regionserver.WAL.metric.not.needed");
+ timelineMetric.setAppId("hbase");
+ Assert.assertFalse(TimelineMetricsFilter.acceptMetric(timelineMetric));
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/timeline/TimelineMetricsIgniteCacheTest.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/timeline/TimelineMetricsIgniteCacheTest.java b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/timeline/TimelineMetricsIgniteCacheTest.java
new file mode 100644
index 0000000..5b47545
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/timeline/TimelineMetricsIgniteCacheTest.java
@@ -0,0 +1,240 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.metrics.core.timeline;
+
+
+import junit.framework.Assert;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.metrics2.sink.timeline.MetricClusterAggregate;
+import org.apache.hadoop.metrics2.sink.timeline.TimelineMetric;
+import org.apache.ambari.metrics.core.timeline.aggregators.TimelineClusterMetric;
+import org.apache.ambari.metrics.core.timeline.discovery.TimelineMetricMetadataManager;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.powermock.core.classloader.annotations.PowerMockIgnore;
+import org.powermock.core.classloader.annotations.PrepareForTest;
+import org.powermock.modules.junit4.PowerMockRunner;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.TreeMap;
+
+import static org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguration.CLUSTER_AGGREGATOR_APP_IDS;
+import static org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguration.HOST_APP_ID;
+import static org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguration.TIMELINE_METRICS_COLLECTOR_IGNITE_NODES;
+import static org.apache.ambari.metrics.core.timeline.aggregators.AggregatorUtils.getRoundedCheckPointTimeMillis;
+import static org.easymock.EasyMock.anyObject;
+import static org.easymock.EasyMock.createNiceMock;
+import static org.easymock.EasyMock.expect;
+import static org.easymock.EasyMock.replay;
+import static org.powermock.api.easymock.PowerMock.mockStatic;
+import static org.powermock.api.easymock.PowerMock.replayAll;
+
+@RunWith(PowerMockRunner.class)
+@PrepareForTest(TimelineMetricConfiguration.class)
+
+@PowerMockIgnore("javax.management.*")
+public class TimelineMetricsIgniteCacheTest {
+ private static TimelineMetricsIgniteCache timelineMetricsIgniteCache;
+ @BeforeClass
+ public static void setupConf() throws Exception {
+ TimelineMetricConfiguration conf = new TimelineMetricConfiguration(new
+ Configuration(), new Configuration());
+ mockStatic(TimelineMetricConfiguration.class);
+ expect(TimelineMetricConfiguration.getInstance()).andReturn(conf).anyTimes();
+ conf.getMetricsConf().set(CLUSTER_AGGREGATOR_APP_IDS, "appIdForHostsAggr");
+ conf.getMetricsConf().set(TIMELINE_METRICS_COLLECTOR_IGNITE_NODES, "localhost");
+ replayAll();
+
+ timelineMetricsIgniteCache = new TimelineMetricsIgniteCache();
+ }
+
+ @Test
+ public void putEvictMetricsFromCacheSlicesMerging() throws Exception {
+ long cacheSliceIntervalMillis = 30000L;
+
+ TimelineMetricMetadataManager metricMetadataManagerMock = createNiceMock(TimelineMetricMetadataManager.class);
+ expect(metricMetadataManagerMock.getUuid(anyObject(TimelineClusterMetric.class))).andReturn(new byte[16]).once();
+ replay(metricMetadataManagerMock);
+
+ long startTime = getRoundedCheckPointTimeMillis(System.currentTimeMillis(), cacheSliceIntervalMillis);
+
+ long seconds = 1000;
+ TreeMap<Long, Double> metricValues = new TreeMap<Long, Double>();
+ /*
+
+ 0 +30s +60s
+ | | |
+ (1)(2)(3) (4)(5)(6) h1
+
+ */
+ // Case 1 : data points are distributed equally, no values are lost, single host.
+ metricValues.put(startTime + 4*seconds, 1.0);
+ metricValues.put(startTime + 14*seconds, 2.0);
+ metricValues.put(startTime + 24*seconds, 3.0);
+ metricValues.put(startTime + 34*seconds, 4.0);
+ metricValues.put(startTime + 44*seconds, 5.0);
+ metricValues.put(startTime + 54*seconds, 6.0);
+
+ TimelineMetric timelineMetric = new TimelineMetric("metric1", "host1", "app1", "instance1");
+ timelineMetric.setStartTime(metricValues.firstKey());
+ timelineMetric.addMetricValues(metricValues);
+
+ Collection<TimelineMetric> timelineMetrics = new ArrayList<>();
+ timelineMetrics.add(timelineMetric);
+ timelineMetricsIgniteCache.putMetrics(timelineMetrics, metricMetadataManagerMock);
+ Map<TimelineClusterMetric, MetricClusterAggregate> aggregateMap = timelineMetricsIgniteCache.evictMetricAggregates(startTime, startTime + 120*seconds);
+
+ Assert.assertEquals(aggregateMap.size(), 2);
+ TimelineClusterMetric timelineClusterMetric = new TimelineClusterMetric(timelineMetric.getMetricName(),
+ timelineMetric.getAppId(), timelineMetric.getInstanceId(), startTime + 30*seconds);
+
+ Assert.assertTrue(aggregateMap.containsKey(timelineClusterMetric));
+ Assert.assertEquals(2.0, aggregateMap.get(timelineClusterMetric).getSum());
+
+ timelineClusterMetric.setTimestamp(startTime + 2*30*seconds);
+ Assert.assertTrue(aggregateMap.containsKey(timelineClusterMetric));
+ Assert.assertEquals(5.0, aggregateMap.get(timelineClusterMetric).getSum());
+
+ metricValues.clear();
+ timelineMetrics.clear();
+
+ /*
+
+ 0 +30s +60s
+ | | |
+ (1)(2)(3) (4)(5)(6) h1, h2
+
+ */
+ // Case 2 : data points are distributed equally, no values are lost, two hosts.
+ metricValues.put(startTime + 4*seconds, 1.0);
+ metricValues.put(startTime + 14*seconds, 2.0);
+ metricValues.put(startTime + 24*seconds, 3.0);
+ metricValues.put(startTime + 34*seconds, 4.0);
+ metricValues.put(startTime + 44*seconds, 5.0);
+ metricValues.put(startTime + 54*seconds, 6.0);
+
+ timelineMetric = new TimelineMetric("metric1", "host1", "app1", "instance1");
+ timelineMetric.setMetricValues(metricValues);
+
+ metricValues = new TreeMap<>();
+ metricValues.put(startTime + 5*seconds, 2.0);
+ metricValues.put(startTime + 15*seconds, 4.0);
+ metricValues.put(startTime + 25*seconds, 6.0);
+ metricValues.put(startTime + 35*seconds, 8.0);
+ metricValues.put(startTime + 45*seconds, 10.0);
+ metricValues.put(startTime + 55*seconds, 12.0);
+ TimelineMetric timelineMetric2 = new TimelineMetric("metric1", "host2", "app1", "instance1");
+ timelineMetric2.setMetricValues(metricValues);
+
+ timelineMetrics = new ArrayList<>();
+ timelineMetrics.add(timelineMetric);
+ timelineMetrics.add(timelineMetric2);
+ timelineMetricsIgniteCache.putMetrics(timelineMetrics, metricMetadataManagerMock);
+ aggregateMap = timelineMetricsIgniteCache.evictMetricAggregates(startTime, startTime + 120*seconds);
+
+ Assert.assertEquals(aggregateMap.size(), 2);
+ timelineClusterMetric = new TimelineClusterMetric(timelineMetric.getMetricName(),
+ timelineMetric.getAppId(), timelineMetric.getInstanceId(), startTime + 30*seconds);
+
+ Assert.assertTrue(aggregateMap.containsKey(timelineClusterMetric));
+ Assert.assertEquals(6.0, aggregateMap.get(timelineClusterMetric).getSum());
+
+ timelineClusterMetric.setTimestamp(startTime + 2*30*seconds);
+ Assert.assertTrue(aggregateMap.containsKey(timelineClusterMetric));
+ Assert.assertEquals(15.0, aggregateMap.get(timelineClusterMetric).getSum());
+
+ metricValues.clear();
+ timelineMetrics.clear();
+
+ Assert.assertEquals(0d, timelineMetricsIgniteCache.getPointInTimeCacheMetrics().get("Cluster_KeySize"));
+ }
+
+ @Test
+ public void updateAppAggregatesFromHostMetricTest() {
+ //make sure hosts metrics are aggregated for appIds from "timeline.metrics.service.cluster.aggregator.appIds"
+
+ long cacheSliceIntervalMillis = 30000L;
+
+ TimelineMetricMetadataManager metricMetadataManagerMock = createNiceMock(TimelineMetricMetadataManager.class);
+ expect(metricMetadataManagerMock.getUuid(anyObject(TimelineClusterMetric.class))).andReturn(new byte[16]).once();
+ expect(metricMetadataManagerMock.getHostedAppsCache()).andReturn(new HashMap<>()).anyTimes();
+ replay(metricMetadataManagerMock);
+
+ long startTime = getRoundedCheckPointTimeMillis(System.currentTimeMillis(), cacheSliceIntervalMillis);
+
+ long seconds = 1000;
+
+ TreeMap<Long, Double> metricValues = new TreeMap<Long, Double>();
+ List<TimelineMetric> timelineMetrics = new ArrayList<>();
+ TimelineMetric timelineMetric;
+
+ metricValues = new TreeMap<>();
+ metricValues.put(startTime + 15*seconds, 1.0);
+ metricValues.put(startTime + 55*seconds, 2.0);
+ timelineMetric = new TimelineMetric("host_metric", "host1", HOST_APP_ID, "instance1");
+ timelineMetric.setMetricValues(metricValues);
+ timelineMetrics.add(timelineMetric);
+
+ metricValues = new TreeMap<>();
+ metricValues.put(startTime + 45*seconds, 3.0);
+ metricValues.put(startTime + 85*seconds, 4.0);
+ timelineMetric = new TimelineMetric("app_metric", "host1", "appIdForHostsAggr", "instance1");
+ timelineMetric.setMetricValues(metricValues);
+ timelineMetrics.add(timelineMetric);
+
+ metricValues = new TreeMap<>();
+ metricValues.put(startTime + 85*seconds, 5.0);
+ timelineMetric = new TimelineMetric("host_metric", "host1", HOST_APP_ID, "instance1");
+ timelineMetric.setMetricValues(metricValues);
+ timelineMetrics.add(timelineMetric);
+
+ metricValues = new TreeMap<>();
+ metricValues.put(startTime + 85*seconds, 6.0);
+ timelineMetric = new TimelineMetric("host_metric", "host2", HOST_APP_ID, "instance1");
+ timelineMetric.setMetricValues(metricValues);
+ timelineMetrics.add(timelineMetric);
+
+ timelineMetricsIgniteCache.putMetrics(timelineMetrics, metricMetadataManagerMock);
+
+ Map<TimelineClusterMetric, MetricClusterAggregate> aggregateMap = timelineMetricsIgniteCache.evictMetricAggregates(startTime, startTime + 120*seconds);
+ Assert.assertEquals(aggregateMap.size(), 6);
+ TimelineClusterMetric timelineClusterMetric = new TimelineClusterMetric(timelineMetric.getMetricName(),
+ timelineMetric.getAppId(), timelineMetric.getInstanceId(), startTime + 90*seconds);
+
+ Assert.assertTrue(aggregateMap.containsKey(timelineClusterMetric));
+ Assert.assertEquals(11.0, aggregateMap.get(timelineClusterMetric).getSum());
+
+ timelineClusterMetric = new TimelineClusterMetric("app_metric",
+ "appIdForHostsAggr", "instance1", startTime + 90*seconds);
+ Assert.assertTrue(aggregateMap.containsKey(timelineClusterMetric));
+ Assert.assertEquals(4.0, aggregateMap.get(timelineClusterMetric).getSum());
+
+ timelineClusterMetric = new TimelineClusterMetric("host_metric",
+ "appIdForHostsAggr", "instance1", startTime + 90*seconds);
+ Assert.assertTrue(aggregateMap.containsKey(timelineClusterMetric));
+ Assert.assertEquals(5.0, aggregateMap.get(timelineClusterMetric).getSum());
+
+ Assert.assertEquals(0d, timelineMetricsIgniteCache.getPointInTimeCacheMetrics().get("Cluster_KeySize"));
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/timeline/TopNConditionTest.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/timeline/TopNConditionTest.java b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/timeline/TopNConditionTest.java
new file mode 100644
index 0000000..5f1c470
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/timeline/TopNConditionTest.java
@@ -0,0 +1,105 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.metrics.core.timeline;
+
+import junit.framework.Assert;
+import org.apache.ambari.metrics.core.timeline.query.TopNCondition;
+import org.junit.Test;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class TopNConditionTest {
+
+
+ @Test
+ public void testTopNCondition() {
+
+ List<String> metricNames = new ArrayList<>();
+ List<String> hostnames = new ArrayList<>();
+
+ //Valid Cases
+
+ // "H" hosts and 1 Metric
+ hostnames.add("h1");
+ hostnames.add("h2");
+ metricNames.add("m1");
+ Assert.assertTrue(TopNCondition.isTopNHostCondition(metricNames, hostnames));
+ Assert.assertFalse(TopNCondition.isTopNMetricCondition(metricNames, hostnames));
+ hostnames.clear();
+
+ // Host(s) with wildcard & 1 Metric
+ hostnames.add("h%");
+ hostnames.add("g1");
+ Assert.assertTrue(TopNCondition.isTopNHostCondition(metricNames, hostnames));
+ Assert.assertFalse(TopNCondition.isTopNMetricCondition(metricNames, hostnames));
+ hostnames.clear();
+
+ // M Metrics and No host
+ metricNames.add("m2");
+ metricNames.add("m3");
+ Assert.assertFalse(TopNCondition.isTopNHostCondition(metricNames, hostnames));
+ Assert.assertTrue(TopNCondition.isTopNMetricCondition(metricNames, hostnames));
+
+ // M Metrics with wildcard and No host
+ metricNames.add("m2");
+ metricNames.add("m%");
+ Assert.assertFalse(TopNCondition.isTopNHostCondition(metricNames, hostnames));
+ Assert.assertTrue(TopNCondition.isTopNMetricCondition(metricNames, hostnames));
+
+ // M Metrics with wildcard and 1 host
+ metricNames.add("m2");
+ metricNames.add("m%");
+ hostnames.add("h1");
+ Assert.assertFalse(TopNCondition.isTopNHostCondition(metricNames, hostnames));
+ Assert.assertTrue(TopNCondition.isTopNMetricCondition(metricNames, hostnames));
+
+ metricNames.clear();
+ hostnames.clear();
+
+ //Invalid Cases
+ // M metrics and H hosts
+ metricNames.add("m1");
+ metricNames.add("m2");
+ hostnames.add("h1");
+ hostnames.add("h2");
+ Assert.assertFalse(TopNCondition.isTopNHostCondition(metricNames, hostnames));
+ Assert.assertFalse(TopNCondition.isTopNMetricCondition(metricNames, hostnames));
+
+ metricNames.clear();
+ hostnames.clear();
+
+ // Wildcard in 1 and multiple in other
+ metricNames.add("m1");
+ metricNames.add("m2");
+ hostnames.add("%");
+ Assert.assertFalse(TopNCondition.isTopNHostCondition(metricNames, hostnames));
+ Assert.assertFalse(TopNCondition.isTopNMetricCondition(metricNames, hostnames));
+
+ metricNames.clear();
+ hostnames.clear();
+
+ //Wildcard in both
+ metricNames.add("m%");
+ hostnames.add("%");
+ Assert.assertFalse(TopNCondition.isTopNHostCondition(metricNames, hostnames));
+ Assert.assertFalse(TopNCondition.isTopNMetricCondition(metricNames, hostnames));
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/timeline/aggregators/AbstractTimelineAggregatorTest.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/timeline/aggregators/AbstractTimelineAggregatorTest.java b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/timeline/aggregators/AbstractTimelineAggregatorTest.java
new file mode 100644
index 0000000..d67ec7f
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/timeline/aggregators/AbstractTimelineAggregatorTest.java
@@ -0,0 +1,181 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.metrics.core.timeline.aggregators;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.ambari.metrics.core.timeline.availability.AggregationTaskRunner;
+import org.apache.ambari.metrics.core.timeline.query.Condition;
+import org.junit.Before;
+import org.junit.Test;
+import java.io.IOException;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.util.concurrent.atomic.AtomicLong;
+
+import static junit.framework.Assert.assertEquals;
+import static org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguration.AGGREGATOR_CHECKPOINT_DELAY;
+import static org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguration.RESULTSET_FETCH_SIZE;
+import static org.apache.ambari.metrics.core.timeline.aggregators.AggregatorUtils.getRoundedAggregateTimeMillis;
+import static org.apache.ambari.metrics.core.timeline.aggregators.AggregatorUtils.getRoundedCheckPointTimeMillis;
+
+public class AbstractTimelineAggregatorTest {
+
+ private AbstractTimelineAggregator agg;
+
+ AtomicLong startTimeInDoWork;
+ AtomicLong endTimeInDoWork;
+ AtomicLong checkPoint;
+ int actualRuns;
+
+ long sleepIntervalMillis;
+ int checkpointCutOffMultiplier;
+
+ @Before
+ public void setUp() throws Exception {
+ sleepIntervalMillis = 5 * 60 * 1000l; //5 minutes
+ checkpointCutOffMultiplier = 2;
+
+ Configuration metricsConf = new Configuration();
+ metricsConf.setInt(AGGREGATOR_CHECKPOINT_DELAY, 0);
+ metricsConf.setInt(RESULTSET_FETCH_SIZE, 2000);
+
+ startTimeInDoWork = new AtomicLong(0);
+ endTimeInDoWork = new AtomicLong(0);
+ checkPoint = new AtomicLong(-1);
+ actualRuns = 0;
+
+ agg = new AbstractTimelineAggregator(AggregationTaskRunner.AGGREGATOR_NAME.METRIC_AGGREGATE_SECOND, null, metricsConf) {
+ @Override
+ public boolean doWork(long startTime, long endTime) {
+ startTimeInDoWork.set(startTime);
+ endTimeInDoWork.set(endTime);
+ actualRuns++;
+
+ return true;
+ }
+
+ @Override
+ protected Condition
+ prepareMetricQueryCondition(long startTime, long endTime) {
+ return null;
+ }
+
+ @Override
+ protected void aggregate(ResultSet rs, long startTime,
+ long endTime) throws IOException, SQLException {
+ }
+
+ @Override
+ public Long getSleepIntervalMillis() {
+ return sleepIntervalMillis;
+ }
+
+ @Override
+ protected Integer getCheckpointCutOffMultiplier() {
+ return checkpointCutOffMultiplier;
+ }
+
+ @Override
+ public boolean isDisabled() {
+ return false;
+ }
+
+ @Override
+ protected String getCheckpointLocation() {
+ return "dummy_ckptFile";
+ }
+
+ protected long readCheckPoint() {
+ return checkPoint.get();
+ }
+
+ @Override
+ protected void saveCheckPoint(long checkpointTime) throws IOException {
+ checkPoint.set(checkpointTime);
+ }
+ };
+
+ }
+
+ @Test
+ public void testDoWorkOnZeroDelay() throws Exception {
+
+ long currentTime = System.currentTimeMillis();
+ long roundedOffAggregatorTime = getRoundedCheckPointTimeMillis(currentTime,
+ sleepIntervalMillis);
+
+ //Test first run of aggregator with no checkpoint
+ checkPoint.set(-1);
+ agg.runOnce(sleepIntervalMillis);
+ assertEquals("startTime should be zero", 0, startTimeInDoWork.get());
+ assertEquals("endTime should be zero", 0, endTimeInDoWork.get());
+ assertEquals(roundedOffAggregatorTime, checkPoint.get());
+ assertEquals("Do not aggregate on first run", 0, actualRuns);
+
+// //Test first run with too "recent" checkpoint
+ currentTime = System.currentTimeMillis();
+ checkPoint.set(currentTime);
+ agg.setSleepIntervalMillis(sleepIntervalMillis);
+ agg.runOnce(sleepIntervalMillis);
+ assertEquals("startTime should be zero", 0, startTimeInDoWork.get());
+ assertEquals("endTime should be zero", 0, endTimeInDoWork.get());
+ assertEquals("Do not aggregate on first run", 0, actualRuns);
+
+ //Test first run with Too Old checkpoint
+ currentTime = System.currentTimeMillis();
+ checkPoint.set(currentTime - 16*60*1000); //Old checkpoint
+ agg.runOnce(sleepIntervalMillis);
+ long checkPointTime = getRoundedAggregateTimeMillis(sleepIntervalMillis);
+ assertEquals("startTime should be zero", checkPointTime - sleepIntervalMillis, startTimeInDoWork.get());
+ assertEquals("endTime should be zero", checkPointTime, endTimeInDoWork.get());
+ assertEquals(roundedOffAggregatorTime, checkPoint.get());
+ assertEquals("Do not aggregate on first run", 1, actualRuns);
+
+
+// //Test first run with perfect checkpoint (sleepIntervalMillis back)
+ currentTime = System.currentTimeMillis();
+ roundedOffAggregatorTime = getRoundedCheckPointTimeMillis(currentTime,
+ sleepIntervalMillis);
+ checkPointTime = roundedOffAggregatorTime - sleepIntervalMillis;
+ long expectedCheckPoint = getRoundedCheckPointTimeMillis(checkPointTime, sleepIntervalMillis);
+ checkPoint.set(checkPointTime);
+ agg.runOnce(sleepIntervalMillis);
+ assertEquals("startTime should the lower rounded time of the checkpoint time",
+ expectedCheckPoint, startTimeInDoWork.get());
+ assertEquals("endTime should the lower rounded time of the checkpoint time + sleepIntervalMillis",
+ expectedCheckPoint + sleepIntervalMillis, endTimeInDoWork.get());
+ assertEquals(expectedCheckPoint + sleepIntervalMillis,
+ checkPoint.get());
+ assertEquals("Aggregate on first run", 2, actualRuns);
+
+ //Test edge case for checkpoint (2 x sleepIntervalMillis)
+ currentTime = System.currentTimeMillis();
+ checkPoint.set(currentTime - 2*sleepIntervalMillis + 5000);
+ agg.runOnce(sleepIntervalMillis);
+ long expectedStartTime = getRoundedCheckPointTimeMillis(currentTime - 2*sleepIntervalMillis + 5000, sleepIntervalMillis);
+ assertEquals("startTime should the lower rounded time of the checkpoint time",
+ expectedStartTime, startTimeInDoWork.get());
+ assertEquals("startTime should the lower rounded time of the checkpoint time + sleepIntervalMillis",
+ expectedStartTime + sleepIntervalMillis, endTimeInDoWork.get());
+ assertEquals(expectedStartTime + sleepIntervalMillis,
+ checkPoint.get());
+ assertEquals("Aggregate on second run", 3, actualRuns);
+
+
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/timeline/aggregators/DownSamplerTest.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/timeline/aggregators/DownSamplerTest.java b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/timeline/aggregators/DownSamplerTest.java
new file mode 100644
index 0000000..f216b13
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/timeline/aggregators/DownSamplerTest.java
@@ -0,0 +1,97 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.metrics.core.timeline.aggregators;
+
+import junit.framework.Assert;
+import org.apache.hadoop.conf.Configuration;
+import org.junit.Ignore;
+import org.junit.Test;
+
+import java.util.List;
+import java.util.Map;
+
+public class DownSamplerTest {
+
+ @Test
+ public void testGetDownsampleMetricPatterns() throws Exception {
+
+ Configuration configuration = new Configuration();
+ configuration.setIfUnset("timeline.metrics.downsampler.topn.metric.patterns", "pattern1,pattern2");
+ configuration.setIfUnset("timeline.metrics.downsampler.lastvalue.metric.patterns", "pattern3");
+
+ List<String> patterns = DownSamplerUtils.getDownsampleMetricPatterns(configuration);
+ Assert.assertEquals(patterns.size(), 3);
+ Assert.assertTrue(patterns.contains("pattern1"));
+ Assert.assertTrue(patterns.contains("pattern2"));
+ Assert.assertTrue(patterns.contains("pattern3"));
+
+ Configuration configuration2 = new Configuration();
+ patterns = DownSamplerUtils.getDownsampleMetricPatterns(configuration2);
+ Assert.assertEquals(patterns.size(), 0);
+ }
+
+ @Test
+ public void testGetDownSamplers() throws Exception {
+
+ Configuration configuration = new Configuration();
+ configuration.setIfUnset("timeline.metrics.downsampler.topn.metric.patterns", "pattern1,pattern2");
+ configuration.setIfUnset("timeline.metrics.downsampler.test.metric.patterns", "pattern3");
+
+ List<CustomDownSampler> downSamplers = DownSamplerUtils.getDownSamplers(configuration);
+ Assert.assertEquals(downSamplers.size(), 1);
+ Assert.assertTrue(downSamplers.get(0) instanceof TopNDownSampler);
+ }
+
+ @Ignore
+ @Test
+ public void testPrepareTopNDownSamplingStatement() throws Exception {
+ Configuration configuration = new Configuration();
+ configuration.setIfUnset("timeline.metrics.downsampler.topn.metric.patterns", "pattern1,pattern2");
+ configuration.setIfUnset("timeline.metrics.downsampler.topn.value", "3");
+
+ Map<String, String> conf = configuration.getValByRegex(DownSamplerUtils.downSamplerConfigPrefix);
+
+ TopNDownSampler topNDownSampler = TopNDownSampler.fromConfig(conf);
+ List<String> stmts = topNDownSampler.prepareDownSamplingStatement(14000000l, 14100000l, "METRIC_RECORD");
+ Assert.assertEquals(stmts.size(),2);
+ Assert.assertTrue(stmts.contains("SELECT METRIC_NAME, HOSTNAME, APP_ID, INSTANCE_ID, 14100000 AS SERVER_TIME, UNITS, " +
+ "MAX(METRIC_MAX), 1, MAX(METRIC_MAX), MAX(METRIC_MAX) FROM METRIC_RECORD WHERE " +
+ "METRIC_NAME LIKE 'pattern1' AND SERVER_TIME > 14000000 AND SERVER_TIME <= 14100000 " +
+ "GROUP BY METRIC_NAME, HOSTNAME, APP_ID, INSTANCE_ID, UNITS ORDER BY MAX(METRIC_MAX) DESC LIMIT 3"));
+
+ Assert.assertTrue(stmts.contains("SELECT METRIC_NAME, HOSTNAME, APP_ID, INSTANCE_ID, 14100000 AS SERVER_TIME, UNITS, " +
+ "MAX(METRIC_MAX), 1, MAX(METRIC_MAX), MAX(METRIC_MAX) FROM METRIC_RECORD WHERE " +
+ "METRIC_NAME LIKE 'pattern2' AND SERVER_TIME > 14000000 AND SERVER_TIME <= 14100000 " +
+ "GROUP BY METRIC_NAME, HOSTNAME, APP_ID, INSTANCE_ID, UNITS ORDER BY MAX(METRIC_MAX) DESC LIMIT 3"));
+
+ configuration.clear();
+ configuration.setIfUnset("timeline.metrics.downsampler.topn.metric.patterns", "pattern1");
+ configuration.setIfUnset("timeline.metrics.downsampler.topn.value", "4");
+ configuration.setIfUnset("timeline.metrics.downsampler.topn.function", "sum");
+ conf = configuration.getValByRegex(DownSamplerUtils.downSamplerConfigPrefix);
+ topNDownSampler = TopNDownSampler.fromConfig(conf);
+ stmts = topNDownSampler.prepareDownSamplingStatement(14000000l, 14100000l, "METRIC_AGGREGATE_MINUTE");
+ Assert.assertEquals(stmts.size(),1);
+
+ Assert.assertTrue(stmts.contains("SELECT METRIC_NAME, APP_ID, INSTANCE_ID, 14100000 AS SERVER_TIME, UNITS, " +
+ "SUM(METRIC_SUM), 1, SUM(METRIC_SUM), SUM(METRIC_SUM) FROM METRIC_AGGREGATE_MINUTE WHERE " +
+ "METRIC_NAME LIKE 'pattern1' AND SERVER_TIME > 14000000 AND SERVER_TIME <= 14100000 " +
+ "GROUP BY METRIC_NAME, APP_ID, INSTANCE_ID, UNITS ORDER BY SUM(METRIC_SUM) DESC LIMIT 4"));
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/42112e28/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/timeline/aggregators/ITClusterAggregator.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/timeline/aggregators/ITClusterAggregator.java b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/timeline/aggregators/ITClusterAggregator.java
new file mode 100644
index 0000000..e01f3b1
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/timeline/aggregators/ITClusterAggregator.java
@@ -0,0 +1,711 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.metrics.core.timeline.aggregators;
+
+
+import static junit.framework.Assert.assertEquals;
+import static junit.framework.Assert.assertNotNull;
+import static junit.framework.Assert.assertTrue;
+import static junit.framework.Assert.fail;
+import static org.apache.ambari.metrics.core.timeline.MetricTestHelper.createEmptyTimelineClusterMetric;
+import static org.apache.ambari.metrics.core.timeline.MetricTestHelper.prepareSingleTimelineMetric;
+import static org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguration.CLUSTER_AGGREGATOR_APP_IDS;
+import static org.apache.ambari.metrics.core.timeline.query.PhoenixTransactSQL.GET_CLUSTER_AGGREGATE_SQL;
+import static org.apache.ambari.metrics.core.timeline.query.PhoenixTransactSQL.METRICS_CLUSTER_AGGREGATE_HOURLY_TABLE_NAME;
+import static org.apache.ambari.metrics.core.timeline.query.PhoenixTransactSQL.METRICS_CLUSTER_AGGREGATE_TABLE_NAME;
+
+import java.sql.Connection;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.TreeMap;
+
+import org.apache.ambari.metrics.core.timeline.AbstractMiniHBaseClusterTest;
+import org.apache.ambari.metrics.core.timeline.MetricTestHelper;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.metrics2.sink.timeline.MetricClusterAggregate;
+import org.apache.hadoop.metrics2.sink.timeline.MetricHostAggregate;
+import org.apache.hadoop.metrics2.sink.timeline.TimelineMetric;
+import org.apache.hadoop.metrics2.sink.timeline.TimelineMetrics;
+import org.apache.ambari.metrics.core.timeline.query.Condition;
+import org.apache.ambari.metrics.core.timeline.query.DefaultCondition;
+import org.apache.ambari.metrics.core.timeline.query.PhoenixTransactSQL;
+import org.junit.Test;
+
+import junit.framework.Assert;
+
+public class ITClusterAggregator extends AbstractMiniHBaseClusterTest {
+ private final TimelineMetricReadHelper metricReader = new TimelineMetricReadHelper(metadataManager, false);
+
+ private Configuration getConfigurationForTest(boolean useGroupByAggregators) {
+ Configuration configuration = new Configuration();
+ configuration.set("timeline.metrics.service.use.groupBy.aggregators", String.valueOf(useGroupByAggregators));
+ return configuration;
+ }
+
+ @Test
+ public void testShouldAggregateClusterProperly() throws Exception {
+ // GIVEN
+ TimelineMetricAggregator agg =
+ TimelineMetricAggregatorFactory.createTimelineClusterAggregatorSecond(hdb,
+ getConfigurationForTest(false), metadataManager, null, null);
+
+ long startTime = System.currentTimeMillis();
+ long ctime = startTime;
+ long minute = 60 * 1000;
+ hdb.insertMetricRecords(MetricTestHelper.prepareSingleTimelineMetric(ctime, "local1",
+ "disk_free", 1));
+ hdb.insertMetricRecords(MetricTestHelper.prepareSingleTimelineMetric(ctime, "local2",
+ "disk_free", 2));
+ ctime += 2*minute;
+ hdb.insertMetricRecords(MetricTestHelper.prepareSingleTimelineMetric(ctime, "local1",
+ "disk_free", 2));
+ hdb.insertMetricRecords(MetricTestHelper.prepareSingleTimelineMetric(ctime, "local2",
+ "disk_free", 1));
+
+ // WHEN
+ long endTime = ctime + minute + 1;
+ boolean success = agg.doWork(startTime, endTime);
+
+ //THEN
+ Condition condition = new DefaultCondition(null, null, null, null, startTime,
+ endTime, null, null, true);
+ condition.setStatement(String.format(GET_CLUSTER_AGGREGATE_SQL,
+ METRICS_CLUSTER_AGGREGATE_TABLE_NAME));
+
+ PreparedStatement pstmt = PhoenixTransactSQL.prepareGetMetricsSqlStmt(conn, condition);
+ ResultSet rs = pstmt.executeQuery();
+
+ int recordCount = 0;
+ while (rs.next()) {
+ TimelineClusterMetric currentMetric = metricReader.fromResultSet(rs);
+ MetricClusterAggregate currentHostAggregate =
+ metricReader.getMetricClusterAggregateFromResultSet(rs);
+
+ if ("disk_free".equals(currentMetric.getMetricName())) {
+ assertEquals(2, currentHostAggregate.getNumberOfHosts());
+ assertEquals(2.0, currentHostAggregate.getMax());
+ assertEquals(1.0, currentHostAggregate.getMin());
+ assertEquals(3.0, currentHostAggregate.getSum());
+ recordCount++;
+ } else {
+ fail("Unexpected entry");
+ }
+ }
+ }
+
+ @Test
+ public void testShouldAggregateClusterIgnoringInstance() throws Exception {
+ // GIVEN
+ TimelineMetricAggregator agg =
+ TimelineMetricAggregatorFactory.createTimelineClusterAggregatorSecond(hdb,
+ getConfigurationForTest(false), metadataManager, null, null);
+ TimelineMetricReadHelper readHelper = new TimelineMetricReadHelper(metadataManager, false);
+
+ long startTime = System.currentTimeMillis();
+ long ctime = startTime;
+ long minute = 60 * 1000 * 2;
+
+ /**
+ * Here we have two nodes with two instances each:
+ * | local1 | local2 |
+ * instance i1 | 1 | 2 |
+ * instance i2 | 3 | 4 |
+ *
+ */
+ // Four 1's at ctime - 100
+ hdb.insertMetricRecords(MetricTestHelper.prepareSingleTimelineMetric(ctime - 100, "local1",
+ "i1", "disk_free", 1), true);
+ // Four 2's at ctime - 100: different host
+ hdb.insertMetricRecords(MetricTestHelper.prepareSingleTimelineMetric(ctime - 100, "local2",
+ "i1", "disk_free", 2), true);
+ // Avoid overwrite
+ hdb.insertMetricRecords(MetricTestHelper.prepareSingleTimelineMetric(ctime + 100, "local1",
+ "i2", "disk_free", 3), true);
+ hdb.insertMetricRecords(MetricTestHelper.prepareSingleTimelineMetric(ctime + 100, "local2",
+ "i2", "disk_free", 4), true);
+
+ ctime += minute;
+
+ // Four 1's at ctime + 2 min
+ hdb.insertMetricRecords(MetricTestHelper.prepareSingleTimelineMetric(ctime - 100, "local1",
+ "i1", "disk_free", 1), true);
+ // Four 1's at ctime + 2 min - different host
+ hdb.insertMetricRecords(MetricTestHelper.prepareSingleTimelineMetric(ctime - 100, "local2",
+ "i1", "disk_free", 3), true);
+ hdb.insertMetricRecords(MetricTestHelper.prepareSingleTimelineMetric(ctime + 100, "local1",
+ "i2", "disk_free", 2), true);
+ hdb.insertMetricRecords(MetricTestHelper.prepareSingleTimelineMetric(ctime + 100, "local2",
+ "i2", "disk_free", 4), true);
+ // WHEN
+ long endTime = ctime + minute;
+ boolean success = agg.doWork(startTime - 1000, endTime + 1000);
+
+ //THEN
+ Condition condition = new DefaultCondition(null, null, null, null, startTime,
+ endTime, null, null, true);
+ condition.setStatement(String.format(GET_CLUSTER_AGGREGATE_SQL,
+ METRICS_CLUSTER_AGGREGATE_TABLE_NAME));
+
+ PreparedStatement pstmt = PhoenixTransactSQL.prepareGetMetricsSqlStmt(conn, condition);
+ ResultSet rs = pstmt.executeQuery();
+
+ int recordCount = 0;
+ while (rs.next()) {
+ TimelineClusterMetric currentMetric = readHelper.fromResultSet(rs);
+ MetricClusterAggregate currentHostAggregate =
+ readHelper.getMetricClusterAggregateFromResultSet(rs);
+
+ if ("disk_free".equals(currentMetric.getMetricName())) {
+ System.out.println("OUTPUT: " + currentMetric + " - " + currentHostAggregate);
+ assertEquals(2, currentHostAggregate.getNumberOfHosts());
+ assertEquals(5.0, Math.floor(currentHostAggregate.getSum()));
+ recordCount++;
+ } else {
+ if (!currentMetric.getMetricName().equals("live_hosts")) {
+ fail("Unexpected entry");
+ }
+ }
+ }
+
+ Assert.assertEquals(6, recordCount); //Interpolation adds 1 record.
+ }
+
+ @Test
+ public void testShouldAggregateDifferentMetricsOnClusterProperly() throws Exception {
+ // GIVEN
+ TimelineMetricAggregator agg =
+ TimelineMetricAggregatorFactory.createTimelineClusterAggregatorSecond(hdb,
+ getConfigurationForTest(false), metadataManager, null, null);
+ TimelineMetricReadHelper readHelper = new TimelineMetricReadHelper(metadataManager, false);
+
+ // here we put some metrics tha will be aggregated
+ long startTime = System.currentTimeMillis();
+ long ctime = startTime;
+ long minute = 60 * 1000;
+ hdb.insertMetricRecords(MetricTestHelper.prepareSingleTimelineMetric(ctime, "local1",
+ "disk_free", 1), true);
+ hdb.insertMetricRecords(MetricTestHelper.prepareSingleTimelineMetric(ctime, "local2",
+ "disk_free", 2), true);
+ hdb.insertMetricRecords(MetricTestHelper.prepareSingleTimelineMetric(ctime, "local1",
+ "disk_used", 1), true);
+
+ ctime += 2*minute;
+ hdb.insertMetricRecords(MetricTestHelper.prepareSingleTimelineMetric(ctime, "local1",
+ "disk_free", 2), true);
+ hdb.insertMetricRecords(MetricTestHelper.prepareSingleTimelineMetric(ctime, "local2",
+ "disk_free", 1), true);
+ hdb.insertMetricRecords(MetricTestHelper.prepareSingleTimelineMetric(ctime, "local1",
+ "disk_used", 1), true);
+
+ // WHEN
+ long endTime = ctime + minute;
+ boolean success = agg.doWork(startTime, endTime);
+
+ //THEN
+ Condition condition = new DefaultCondition(null, null, null, null, startTime,
+ endTime, null, null, true);
+ condition.setStatement(String.format(GET_CLUSTER_AGGREGATE_SQL,
+ METRICS_CLUSTER_AGGREGATE_TABLE_NAME));
+
+ PreparedStatement pstmt = PhoenixTransactSQL.prepareGetMetricsSqlStmt(conn, condition);
+ ResultSet rs = pstmt.executeQuery();
+
+ int recordCount = 0;
+ while (rs.next()) {
+ TimelineClusterMetric currentMetric = readHelper.fromResultSet(rs);
+ MetricClusterAggregate currentHostAggregate =
+ readHelper.getMetricClusterAggregateFromResultSet(rs);
+
+ if ("disk_free".equals(currentMetric.getMetricName())) {
+ assertEquals(2, currentHostAggregate.getNumberOfHosts());
+ assertEquals(2.0, currentHostAggregate.getMax());
+ assertEquals(1.0, currentHostAggregate.getMin());
+ assertEquals(3.0, currentHostAggregate.getSum());
+ recordCount++;
+ } else if ("disk_used".equals(currentMetric.getMetricName())) {
+ assertEquals(1, currentHostAggregate.getNumberOfHosts());
+ assertEquals(1.0, currentHostAggregate.getMax());
+ assertEquals(1.0, currentHostAggregate.getMin());
+ assertEquals(1.0, currentHostAggregate.getSum());
+ recordCount++;
+ } else {
+ if (!currentMetric.getMetricName().equals("live_hosts")) {
+ fail("Unexpected entry");
+ }
+ }
+ }
+ }
+
+ @Test
+ public void testAggregateDailyClusterMetrics() throws Exception {
+ // GIVEN
+ TimelineMetricAggregator agg =
+ TimelineMetricAggregatorFactory.createTimelineClusterAggregatorDaily(hdb, getConfigurationForTest(
+ false),
+ metadataManager,
+ null);
+
+ // this time can be virtualized! or made independent from real clock
+ long startTime = System.currentTimeMillis();
+ long ctime = startTime;
+ long hour = 3600 * 1000;
+
+ Map<TimelineClusterMetric, MetricHostAggregate> records =
+ new HashMap<TimelineClusterMetric, MetricHostAggregate>();
+
+ records.put(MetricTestHelper.createEmptyTimelineClusterMetric(ctime),
+ MetricTestHelper.createMetricHostAggregate(4.0, 0.0, 2, 4.0));
+ records.put(MetricTestHelper.createEmptyTimelineClusterMetric(ctime += hour),
+ MetricTestHelper.createMetricHostAggregate(4.0, 0.0, 2, 4.0));
+ records.put(MetricTestHelper.createEmptyTimelineClusterMetric(ctime += hour),
+ MetricTestHelper.createMetricHostAggregate(4.0, 0.0, 2, 4.0));
+ records.put(MetricTestHelper.createEmptyTimelineClusterMetric(ctime += hour),
+ MetricTestHelper.createMetricHostAggregate(4.0, 0.0, 2, 4.0));
+
+
+ hdb.saveClusterAggregateRecordsSecond(records, METRICS_CLUSTER_AGGREGATE_HOURLY_TABLE_NAME);
+
+ // WHEN
+ agg.doWork(startTime, ctime + hour + 1000);
+
+ // THEN
+ ResultSet rs = executeQuery("SELECT * FROM METRIC_AGGREGATE_DAILY");
+ int count = 0;
+ while (rs.next()) {
+ TimelineMetric metric = metadataManager.getMetricFromUuid(rs.getBytes("UUID"));
+ assertEquals("METRIC_NAME", "disk_used", metric.getMetricName());
+ assertEquals("APP_ID", "test_app", metric.getAppId());
+ assertEquals("METRIC_SUM", 16.0, rs.getDouble("METRIC_SUM"));
+ assertEquals("METRIC_COUNT", 8, rs.getLong("METRIC_COUNT"));
+ assertEquals("METRIC_MAX", 4.0, rs.getDouble("METRIC_MAX"));
+ assertEquals("METRIC_MIN", 0.0, rs.getDouble("METRIC_MIN"));
+ count++;
+ }
+
+ assertEquals("Day aggregated row expected ", 1, count);
+ }
+
+ @Test
+ public void testShouldAggregateClusterOnMinuteProperly() throws Exception {
+
+ TimelineMetricAggregator agg =
+ TimelineMetricAggregatorFactory.createTimelineClusterAggregatorMinute(
+ hdb,
+ getConfigurationForTest(false),
+ metadataManager,
+ null);
+
+ long startTime = System.currentTimeMillis();
+ long ctime = startTime;
+ long second = 1000;
+ long minute = 60*second;
+
+ Map<TimelineClusterMetric, MetricClusterAggregate> records =
+ new HashMap<TimelineClusterMetric, MetricClusterAggregate>();
+
+ records.put(MetricTestHelper.createEmptyTimelineClusterMetric(ctime),
+ new MetricClusterAggregate(4.0, 2, 0.0, 4.0, 0.0));
+ records.put(MetricTestHelper.createEmptyTimelineClusterMetric(ctime += second),
+ new MetricClusterAggregate(4.0, 2, 0.0, 4.0, 0.0));
+ records.put(MetricTestHelper.createEmptyTimelineClusterMetric(ctime += second),
+ new MetricClusterAggregate(4.0, 2, 0.0, 4.0, 0.0));
+ records.put(MetricTestHelper.createEmptyTimelineClusterMetric(ctime += second),
+ new MetricClusterAggregate(4.0, 2, 0.0, 4.0, 0.0));
+
+ hdb.saveClusterAggregateRecords(records);
+ agg.doWork(startTime, ctime + second);
+ long oldCtime = ctime + second;
+
+ //Next minute
+ ctime = startTime + minute;
+
+ records.put(MetricTestHelper.createEmptyTimelineClusterMetric(ctime),
+ new MetricClusterAggregate(4.0, 2, 0.0, 4.0, 0.0));
+ records.put(MetricTestHelper.createEmptyTimelineClusterMetric(ctime += second),
+ new MetricClusterAggregate(4.0, 2, 0.0, 4.0, 0.0));
+ records.put(MetricTestHelper.createEmptyTimelineClusterMetric(ctime += second),
+ new MetricClusterAggregate(4.0, 2, 0.0, 4.0, 0.0));
+ records.put(MetricTestHelper.createEmptyTimelineClusterMetric(ctime += second),
+ new MetricClusterAggregate(4.0, 2, 0.0, 4.0, 0.0));
+
+ hdb.saveClusterAggregateRecords(records);
+ agg.doWork(oldCtime, ctime + second);
+
+ ResultSet rs = executeQuery("SELECT * FROM METRIC_AGGREGATE_MINUTE");
+ int count = 0;
+ long diff = 0 ;
+ while (rs.next()) {
+ TimelineMetric metric = metadataManager.getMetricFromUuid(rs.getBytes("UUID"));
+ assertEquals("METRIC_NAME", "disk_used", metric.getMetricName());
+ assertEquals("APP_ID", "test_app", metric.getAppId());
+ assertEquals("METRIC_SUM", 16.0, rs.getDouble("METRIC_SUM"));
+ assertEquals("METRIC_COUNT", 8, rs.getLong("METRIC_COUNT"));
+ assertEquals("METRIC_MAX", 4.0, rs.getDouble("METRIC_MAX"));
+ assertEquals("METRIC_MIN", 0.0, rs.getDouble("METRIC_MIN"));
+ if (count == 0) {
+ diff+=rs.getLong("SERVER_TIME");
+ } else {
+ diff-=rs.getLong("SERVER_TIME");
+ if (diff < 0) {
+ diff*=-1;
+ }
+ assertTrue(diff == minute);
+ }
+ count++;
+ }
+
+ assertEquals("One hourly aggregated row expected ", 2, count);
+ }
+
+ @Test
+ public void testShouldAggregateClusterOnHourProperly() throws Exception {
+ // GIVEN
+ TimelineMetricAggregator agg =
+ TimelineMetricAggregatorFactory.createTimelineClusterAggregatorHourly(
+ hdb,
+ getConfigurationForTest(false),
+ metadataManager,
+ null);
+
+ // this time can be virtualized! or made independent from real clock
+ long startTime = System.currentTimeMillis();
+ long ctime = startTime;
+ long minute = 60 * 1000;
+
+ Map<TimelineClusterMetric, MetricClusterAggregate> records =
+ new HashMap<TimelineClusterMetric, MetricClusterAggregate>();
+
+ records.put(MetricTestHelper.createEmptyTimelineClusterMetric(ctime),
+ new MetricClusterAggregate(4.0, 2, 0.0, 4.0, 0.0));
+ records.put(MetricTestHelper.createEmptyTimelineClusterMetric(ctime += minute),
+ new MetricClusterAggregate(4.0, 2, 0.0, 4.0, 0.0));
+ records.put(MetricTestHelper.createEmptyTimelineClusterMetric(ctime += minute),
+ new MetricClusterAggregate(4.0, 2, 0.0, 4.0, 0.0));
+ records.put(MetricTestHelper.createEmptyTimelineClusterMetric(ctime += minute),
+ new MetricClusterAggregate(4.0, 2, 0.0, 4.0, 0.0));
+
+ hdb.saveClusterAggregateRecords(records);
+
+ // WHEN
+ agg.doWork(startTime, ctime + minute);
+
+ // THEN
+ ResultSet rs = executeQuery("SELECT * FROM METRIC_AGGREGATE_HOURLY");
+ int count = 0;
+ while (rs.next()) {
+ TimelineMetric metric = metadataManager.getMetricFromUuid(rs.getBytes("UUID"));
+ assertEquals("METRIC_NAME", "disk_used", metric.getMetricName());
+ assertEquals("APP_ID", "test_app", metric.getAppId());
+ assertEquals("METRIC_SUM", 16.0, rs.getDouble("METRIC_SUM"));
+ assertEquals("METRIC_COUNT", 8, rs.getLong("METRIC_COUNT"));
+ assertEquals("METRIC_MAX", 4.0, rs.getDouble("METRIC_MAX"));
+ assertEquals("METRIC_MIN", 0.0, rs.getDouble("METRIC_MIN"));
+ count++;
+ }
+
+ assertEquals("One hourly aggregated row expected ", 1, count);
+ }
+
+ @Test
+ public void testShouldAggregateDifferentMetricsOnHourProperly() throws Exception {
+ // GIVEN
+ TimelineMetricAggregator agg =
+ TimelineMetricAggregatorFactory.createTimelineClusterAggregatorHourly(
+ hdb,
+ getConfigurationForTest(false),
+ metadataManager,
+ null);
+
+ long startTime = System.currentTimeMillis();
+ long ctime = startTime;
+ long minute = 60 * 1000;
+
+ Map<TimelineClusterMetric, MetricClusterAggregate> records =
+ new HashMap<TimelineClusterMetric, MetricClusterAggregate>();
+
+ records.put(MetricTestHelper.createEmptyTimelineClusterMetric("disk_used", ctime),
+ new MetricClusterAggregate(4.0, 2, 0.0, 4.0, 0.0));
+ records.put(MetricTestHelper.createEmptyTimelineClusterMetric("disk_free", ctime),
+ new MetricClusterAggregate(1.0, 2, 0.0, 1.0, 1.0));
+
+ records.put(MetricTestHelper.createEmptyTimelineClusterMetric("disk_used", ctime += minute),
+ new MetricClusterAggregate(4.0, 2, 0.0, 4.0, 0.0));
+ records.put(MetricTestHelper.createEmptyTimelineClusterMetric("disk_free", ctime),
+ new MetricClusterAggregate(1.0, 2, 0.0, 1.0, 1.0));
+
+ records.put(MetricTestHelper.createEmptyTimelineClusterMetric("disk_used", ctime += minute),
+ new MetricClusterAggregate(4.0, 2, 0.0, 4.0, 0.0));
+ records.put(MetricTestHelper.createEmptyTimelineClusterMetric("disk_free", ctime),
+ new MetricClusterAggregate(1.0, 2, 0.0, 1.0, 1.0));
+
+ records.put(MetricTestHelper.createEmptyTimelineClusterMetric("disk_used", ctime += minute),
+ new MetricClusterAggregate(4.0, 2, 0.0, 4.0, 0.0));
+ records.put(MetricTestHelper.createEmptyTimelineClusterMetric("disk_free", ctime),
+ new MetricClusterAggregate(1.0, 2, 0.0, 1.0, 1.0));
+
+ hdb.saveClusterAggregateRecords(records);
+
+ // WHEN
+ agg.doWork(startTime, ctime + minute);
+
+ // THEN
+ ResultSet rs = executeQuery("SELECT * FROM METRIC_AGGREGATE_HOURLY");
+ int count = 0;
+ while (rs.next()) {
+ TimelineMetric metric = metadataManager.getMetricFromUuid(rs.getBytes("UUID"));
+ if ("disk_used".equals(metric.getMetricName())) {
+ assertEquals("APP_ID", "test_app", metric.getAppId());
+ assertEquals("METRIC_SUM", 4.0, rs.getDouble("METRIC_SUM"));
+ assertEquals("METRIC_COUNT", 2, rs.getLong("METRIC_COUNT"));
+ assertEquals("METRIC_MAX", 4.0, rs.getDouble("METRIC_MAX"));
+ assertEquals("METRIC_MIN", 0.0, rs.getDouble("METRIC_MIN"));
+ } else if ("disk_free".equals(metric.getMetricName())) {
+ assertEquals("APP_ID", "test_app", metric.getAppId());
+ assertEquals("METRIC_SUM", 1.0, rs.getDouble("METRIC_SUM"));
+ assertEquals("METRIC_COUNT", 2, rs.getLong("METRIC_COUNT"));
+ assertEquals("METRIC_MAX", 1.0, rs.getDouble("METRIC_MAX"));
+ assertEquals("METRIC_MIN", 1.0, rs.getDouble("METRIC_MIN"));
+ }
+
+ count++;
+ }
+
+ assertEquals("Two hourly aggregated row expected ", 2, count);
+ }
+
+ @Test
+ public void testAppLevelHostMetricAggregates() throws Exception {
+ Configuration conf = getConfigurationForTest(false);
+ conf.set(CLUSTER_AGGREGATOR_APP_IDS, "app1");
+ TimelineMetricAggregator agg =
+ TimelineMetricAggregatorFactory.createTimelineClusterAggregatorSecond(
+ hdb,
+ conf,
+ metadataManager,
+ null,
+ null);
+ TimelineMetricReadHelper readHelper = new TimelineMetricReadHelper(metadataManager, false);
+
+ long startTime = System.currentTimeMillis();
+ long ctime = startTime;
+ long minute = 60 * 1000;
+ hdb.insertMetricRecords(MetricTestHelper.prepareSingleTimelineMetric((ctime), "local1",
+ "app1", null, "app_metric_random", 1), true);
+ ctime += 10;
+ hdb.insertMetricRecords(MetricTestHelper.prepareSingleTimelineMetric(ctime, "local1",
+ "cpu_user", 1), true);
+ ctime += 10;
+ hdb.insertMetricRecords(MetricTestHelper.prepareSingleTimelineMetric(ctime, "local2",
+ "cpu_user", 2), true);
+
+ // WHEN
+ long endTime = ctime + minute;
+ boolean success = agg.doWork(startTime, endTime);
+
+ //THEN
+ List<byte[]> uuids = metadataManager.getUuids(new ArrayList<String>() {{ add("cpu_user"); }},
+ null,
+ "app1", null);
+
+ Condition condition = new DefaultCondition(uuids,
+ new ArrayList<String>() {{ add("cpu_user"); }}, null, "app1", null,
+ startTime - 90000, endTime, null, null, true);
+ condition.setStatement(String.format(GET_CLUSTER_AGGREGATE_SQL,
+ METRICS_CLUSTER_AGGREGATE_TABLE_NAME));
+
+ PreparedStatement pstmt = PhoenixTransactSQL.prepareGetMetricsSqlStmt(conn, condition);
+ ResultSet rs = pstmt.executeQuery();
+
+ int recordCount = 0;
+ TimelineClusterMetric currentMetric = null;
+ MetricClusterAggregate currentHostAggregate = null;
+ while (rs.next()) {
+ currentMetric = readHelper.fromResultSet(rs);
+ currentHostAggregate = readHelper.getMetricClusterAggregateFromResultSet(rs);
+ recordCount++;
+ }
+ assertEquals(3, recordCount);
+ assertNotNull(currentMetric);
+ assertEquals("cpu_user", currentMetric.getMetricName());
+ assertEquals("app1", currentMetric.getAppId());
+ assertNotNull(currentHostAggregate);
+ assertEquals(1, currentHostAggregate.getNumberOfHosts());
+ assertEquals(1.0d, currentHostAggregate.getSum());
+ }
+
+ @Test
+ public void testClusterAggregateMetricNormalization() throws Exception {
+ TimelineMetricAggregator agg =
+ TimelineMetricAggregatorFactory.createTimelineClusterAggregatorSecond(
+ hdb,
+ getConfigurationForTest(false),
+ metadataManager,
+ null,
+ null);
+ TimelineMetricReadHelper readHelper = new TimelineMetricReadHelper(metadataManager, false);
+
+ long currentTime = System.currentTimeMillis();
+ // Sample data
+ TimelineMetric metric1 = new TimelineMetric();
+ metric1.setMetricName("yarn.ClusterMetrics.NumActiveNMs");
+ metric1.setAppId("resourcemanager");
+ metric1.setHostName("h1");
+ metric1.setStartTime(currentTime);
+ metric1.setMetricValues(new TreeMap<Long, Double>() {{
+ put(currentTime + 10000, 1.0);
+ put(currentTime + 20000, 1.0);
+ put(currentTime + 30000, 1.0);
+ put(currentTime + 40000, 1.0);
+ put(currentTime + 50000, 1.0);
+ put(currentTime + 60000, 1.0);
+ put(currentTime + 70000, 1.0);
+ }});
+
+ TimelineMetric metric2 = new TimelineMetric();
+ metric2.setMetricName("yarn.ClusterMetrics.NumActiveNMs");
+ metric2.setAppId("resourcemanager");
+ metric2.setHostName("h1");
+ metric2.setStartTime(currentTime + 70000);
+ metric2.setMetricValues(new TreeMap<Long, Double>() {{
+ put(currentTime + 70000, 1.0);
+ put(currentTime + 80000, 1.0);
+ put(currentTime + 90000, 1.0);
+ put(currentTime + 100000, 1.0);
+ put(currentTime + 110000, 1.0);
+ put(currentTime + 120000, 1.0);
+ put(currentTime + 130000, 1.0);
+ }});
+
+ TimelineMetrics metrics = new TimelineMetrics();
+ metrics.setMetrics(Collections.singletonList(metric1));
+ insertMetricRecords(conn, metrics);
+
+ metrics.setMetrics(Collections.singletonList(metric2));
+ insertMetricRecords(conn, metrics);
+
+ long startTime = currentTime - 3*60*1000;
+ long endTime = currentTime + 3*60*1000;
+
+ agg.doWork(startTime, endTime);
+
+ Condition condition = new DefaultCondition(null, null, null, null, startTime,
+ endTime, null, null, true);
+ condition.setStatement(String.format(GET_CLUSTER_AGGREGATE_SQL,
+ METRICS_CLUSTER_AGGREGATE_TABLE_NAME));
+
+ PreparedStatement pstmt = PhoenixTransactSQL.prepareGetMetricsSqlStmt(conn, condition);
+ ResultSet rs = pstmt.executeQuery();
+
+ int recordCount = 0;
+ while (rs.next()) {
+ TimelineClusterMetric currentMetric = readHelper.fromResultSet(rs);
+ MetricClusterAggregate currentHostAggregate = readHelper.getMetricClusterAggregateFromResultSet(rs);
+
+ if ("yarn.ClusterMetrics.NumActiveNMs".equals(currentMetric.getMetricName())) {
+ assertEquals(1, currentHostAggregate.getNumberOfHosts());
+ assertEquals(1.0, currentHostAggregate.getMax());
+ assertEquals(1.0, currentHostAggregate.getMin());
+ assertEquals(1.0, currentHostAggregate.getSum());
+ recordCount++;
+ } else {
+ if (!currentMetric.getMetricName().equals("live_hosts")) {
+ fail("Unexpected entry");
+ }
+ }
+ }
+ Assert.assertEquals(10, recordCount); //With interpolation.
+ }
+
+ @Test
+ public void testAggregationUsingGroupByQuery() throws Exception {
+ // GIVEN
+ TimelineMetricAggregator agg =
+ TimelineMetricAggregatorFactory.createTimelineClusterAggregatorHourly(
+ hdb,
+ getConfigurationForTest(true),
+ metadataManager,
+ null);
+
+ long startTime = System.currentTimeMillis();
+ long ctime = startTime;
+ long minute = 60 * 1000;
+
+ Map<TimelineClusterMetric, MetricClusterAggregate> records =
+ new HashMap<TimelineClusterMetric, MetricClusterAggregate>();
+
+ records.put(MetricTestHelper.createEmptyTimelineClusterMetric("disk_used", ctime += minute),
+ new MetricClusterAggregate(4.0, 2, 0.0, 4.0, 0.0));
+ records.put(MetricTestHelper.createEmptyTimelineClusterMetric("disk_free", ctime),
+ new MetricClusterAggregate(1.0, 2, 0.0, 1.0, 1.0));
+
+ records.put(MetricTestHelper.createEmptyTimelineClusterMetric("disk_used", ctime += minute),
+ new MetricClusterAggregate(4.0, 2, 0.0, 4.0, 0.0));
+ records.put(MetricTestHelper.createEmptyTimelineClusterMetric("disk_free", ctime),
+ new MetricClusterAggregate(1.0, 2, 0.0, 1.0, 1.0));
+
+ records.put(MetricTestHelper.createEmptyTimelineClusterMetric("disk_used", ctime += minute),
+ new MetricClusterAggregate(4.0, 2, 0.0, 4.0, 0.0));
+ records.put(MetricTestHelper.createEmptyTimelineClusterMetric("disk_free", ctime),
+ new MetricClusterAggregate(1.0, 2, 0.0, 1.0, 1.0));
+
+ records.put(MetricTestHelper.createEmptyTimelineClusterMetric("disk_used", ctime += minute),
+ new MetricClusterAggregate(4.0, 2, 0.0, 4.0, 0.0));
+ records.put(MetricTestHelper.createEmptyTimelineClusterMetric("disk_free", ctime),
+ new MetricClusterAggregate(1.0, 2, 0.0, 1.0, 1.0));
+
+ hdb.saveClusterAggregateRecords(records);
+
+ // WHEN
+ agg.doWork(startTime, ctime + minute);
+
+ // THEN
+ ResultSet rs = executeQuery("SELECT * FROM METRIC_AGGREGATE_HOURLY");
+ int count = 0;
+ while (rs.next()) {
+ TimelineMetric metric = metadataManager.getMetricFromUuid(rs.getBytes("UUID"));
+ if ("disk_used".equals(metric.getMetricName())) {
+ assertEquals("APP_ID", "test_app", metric.getAppId());
+ assertEquals("METRIC_SUM", 4.0, rs.getDouble("METRIC_SUM"));
+ assertEquals("METRIC_COUNT", 2, rs.getLong("METRIC_COUNT"));
+ assertEquals("METRIC_MAX", 4.0, rs.getDouble("METRIC_MAX"));
+ assertEquals("METRIC_MIN", 0.0, rs.getDouble("METRIC_MIN"));
+ } else if ("disk_free".equals(metric.getMetricName())) {
+ assertEquals("APP_ID", "test_app", metric.getAppId());
+ assertEquals("METRIC_SUM", 1.0, rs.getDouble("METRIC_SUM"));
+ assertEquals("METRIC_COUNT", 2, rs.getLong("METRIC_COUNT"));
+ assertEquals("METRIC_MAX", 1.0, rs.getDouble("METRIC_MAX"));
+ assertEquals("METRIC_MIN", 1.0, rs.getDouble("METRIC_MIN"));
+ }
+ count++;
+ }
+ assertEquals("Two hourly aggregated row expected ", 2, count);
+ }
+
+ private ResultSet executeQuery(String query) throws SQLException {
+ Connection conn = getConnection(getUrl());
+ Statement stmt = conn.createStatement();
+ return stmt.executeQuery(query);
+ }
+}