You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by sw...@apache.org on 2015/11/17 23:58:50 UTC
[4/4] ambari git commit: AMBARI-13926. AMS log file not getting
updated or created on fresh deploy. (Aravindan Vijayan via swagle)
AMBARI-13926. AMS log file not getting updated or created on fresh deploy. (Aravindan Vijayan via swagle)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/05a4b66b
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/05a4b66b
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/05a4b66b
Branch: refs/heads/branch-2.1
Commit: 05a4b66bd1d3ca0ba7b2a42e7df7f0872986d82f
Parents: 9cf3bca
Author: Siddharth Wagle <sw...@hortonworks.com>
Authored: Tue Nov 17 14:58:26 2015 -0800
Committer: Siddharth Wagle <sw...@hortonworks.com>
Committed: Tue Nov 17 14:58:26 2015 -0800
----------------------------------------------------------------------
.../ambari-metrics-timelineservice/pom.xml | 2 +
.../jmetertest/AMSJMeterLoadTest.java | 202 ---
.../loadsimulator/jmetertest/AppGetMetric.java | 57 -
.../jmetertest/GetMetricRequestInfo.java | 61 -
.../jmetertest/JmeterTestPlanTask.java | 269 ----
.../src/main/resources/loadsimulator/README | 65 -
.../loadsimulator/ams-jmeter.properties | 56 -
.../resources/loadsimulator/amsJmeterGraph.jmx | 104 --
.../resources/loadsimulator/jmeter.properties | 1172 ------------------
.../loadsimulator/saveservice.properties | 381 ------
.../main/resources/ui_metrics_def/AMS-HBASE.dat | 26 -
.../main/resources/ui_metrics_def/DATANODE.dat | 4 -
.../resources/ui_metrics_def/FLUME_HANDLER.dat | 63 -
.../src/main/resources/ui_metrics_def/HBASE.dat | 47 -
.../src/main/resources/ui_metrics_def/HOST.dat | 79 --
.../resources/ui_metrics_def/KAFKA_BROKER.dat | 16 -
.../main/resources/ui_metrics_def/NAMENODE.dat | 30 -
.../main/resources/ui_metrics_def/NIMBUS.dat | 28 -
.../resources/ui_metrics_def/NODEMANAGER.dat | 33 -
.../ui_metrics_def/RESOURCEMANAGER.dat | 11 -
.../jmetertest/AMSJMeterLoadTest.java | 202 +++
.../jmetertest/jmetertest/AppGetMetric.java | 57 +
.../jmetertest/GetMetricRequestInfo.java | 61 +
.../jmetertest/JmeterTestPlanTask.java | 276 +++++
.../src/test/resources/loadsimulator/README | 65 +
.../loadsimulator/ams-jmeter.properties | 56 +
.../resources/loadsimulator/amsJmeterGraph.jmx | 104 ++
.../resources/loadsimulator/jmeter.properties | 1172 ++++++++++++++++++
.../loadsimulator/saveservice.properties | 381 ++++++
.../test/resources/ui_metrics_def/AMS-HBASE.dat | 26 +
.../test/resources/ui_metrics_def/DATANODE.dat | 4 +
.../resources/ui_metrics_def/FLUME_HANDLER.dat | 63 +
.../src/test/resources/ui_metrics_def/HBASE.dat | 47 +
.../src/test/resources/ui_metrics_def/HOST.dat | 79 ++
.../resources/ui_metrics_def/KAFKA_BROKER.dat | 16 +
.../test/resources/ui_metrics_def/NAMENODE.dat | 30 +
.../test/resources/ui_metrics_def/NIMBUS.dat | 28 +
.../resources/ui_metrics_def/NODEMANAGER.dat | 33 +
.../ui_metrics_def/RESOURCEMANAGER.dat | 11 +
39 files changed, 2713 insertions(+), 2704 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/05a4b66b/ambari-metrics/ambari-metrics-timelineservice/pom.xml
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/pom.xml b/ambari-metrics/ambari-metrics-timelineservice/pom.xml
index 1f76ba2..7b4afb9 100644
--- a/ambari-metrics/ambari-metrics-timelineservice/pom.xml
+++ b/ambari-metrics/ambari-metrics-timelineservice/pom.xml
@@ -413,6 +413,7 @@
<groupId>org.apache.jmeter</groupId>
<artifactId>ApacheJMeter_core</artifactId>
<version>2.13</version>
+ <scope>test</scope>
<exclusions>
<exclusion>
<groupId>org.slf4j</groupId>
@@ -432,6 +433,7 @@
<groupId>org.apache.jmeter</groupId>
<artifactId>ApacheJMeter_http</artifactId>
<version>2.13</version>
+ <scope>test</scope>
<exclusions>
<exclusion>
<groupId>org.slf4j</groupId>
http://git-wip-us.apache.org/repos/asf/ambari/blob/05a4b66b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/jmetertest/AMSJMeterLoadTest.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/jmetertest/AMSJMeterLoadTest.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/jmetertest/AMSJMeterLoadTest.java
deleted file mode 100644
index 187c3f1..0000000
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/jmetertest/AMSJMeterLoadTest.java
+++ /dev/null
@@ -1,202 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p/>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p/>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.loadsimulator.jmetertest;
-
-import org.apache.commons.lang3.StringUtils;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.loadsimulator.MetricsLoadSimulator;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.BufferedReader;
-import java.io.FileInputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Properties;
-import java.util.concurrent.Executors;
-import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.ScheduledFuture;
-import java.util.concurrent.TimeUnit;
-
-public class AMSJMeterLoadTest {
-
- private final static Logger LOG = LoggerFactory.getLogger(AMSJMeterLoadTest.class);
- private static String PROPERTIES_FILE = "loadsimulator/ams-jmeter.properties";
- private ScheduledExecutorService scheduledExecutorService = null;
- private List<AppGetMetric> appGetMetrics;
- private Properties amsJmeterProperties = null;
-
- public AMSJMeterLoadTest(String testType, String userDefinedPropertiesFile) {
-
- if (null == userDefinedPropertiesFile || userDefinedPropertiesFile.isEmpty()) {
- this.amsJmeterProperties = readProperties(PROPERTIES_FILE);
- } else {
- this.amsJmeterProperties = readProperties(userDefinedPropertiesFile);
- }
-
- if ("U".equals(testType)) { //GET metrics simulator
- int numInstances = Integer.valueOf(amsJmeterProperties.getProperty("num-ui-instances"));
- this.scheduledExecutorService = Executors.newScheduledThreadPool(numInstances);
- this.appGetMetrics = initializeGetMetricsPayload(amsJmeterProperties);
- this.runTest(numInstances);
- } else { //PUT Metrics simulator
- Map<String, String> mapArgs = new HashMap<String, String>();
- mapArgs.put("hostName", amsJmeterProperties.getProperty("host-prefix"));
- mapArgs.put("numberOfHosts", amsJmeterProperties.getProperty("num-hosts"));
- mapArgs.put("metricsHostName", amsJmeterProperties.getProperty("ams-host-port"));
- mapArgs.put("collectInterval", amsJmeterProperties.getProperty("collection-interval"));
- mapArgs.put("sendInterval", amsJmeterProperties.getProperty("send-interval"));
- mapArgs.put("master", amsJmeterProperties.getProperty("create-master"));
- MetricsLoadSimulator.startTest(mapArgs);
- }
- }
-
- public static Properties readProperties(String propertiesFile) {
- try {
- Properties properties = new Properties();
- InputStream inputStream = ClassLoader.getSystemResourceAsStream(propertiesFile);
- if (inputStream == null) {
- inputStream = new FileInputStream(propertiesFile);
- }
- properties.load(inputStream);
- return properties;
- } catch (IOException ioEx) {
- LOG.error("Error reading properties file for jmeter");
- return null;
- }
- }
-
- private static List<GetMetricRequestInfo> readMetricsFromFile(String app) {
- InputStream input = null;
- List<GetMetricRequestInfo> metricList = new ArrayList<>();
- String fileName = "ui_metrics_def/" + app + ".dat";
-
- try {
- input = ClassLoader.getSystemResourceAsStream(fileName);
- BufferedReader reader = new BufferedReader(new InputStreamReader(input));
- String line;
- List<String> metrics = new ArrayList<>();
- while ((line = reader.readLine()) != null) {
-
- if (line.startsWith("|")) {
- boolean needsTimestamps = line.contains("startTime");
- boolean needsHost = line.contains("hostname");
- metricList.add(new GetMetricRequestInfo(metrics, needsTimestamps, needsHost));
- metrics.clear();
- } else {
- metrics.add(line);
- }
- }
- return metricList;
- } catch (IOException e) {
- LOG.error("Cannot read file " + fileName + " for appID " + app, e);
- } finally {
- if (input != null) {
- try {
- input.close();
- } catch (IOException ex) {
- }
- }
- }
- return null;
- }
-
- private static List<AppGetMetric> initializeGetMetricsPayload(Properties amsJmeterProperties) {
-
- List<AppGetMetric> appGetMetrics = new ArrayList<AppGetMetric>();
- String appsToTest = amsJmeterProperties.getProperty("apps-to-test");
- String[] apps;
-
- if (appsToTest != null && !appsToTest.isEmpty()) {
- apps = StringUtils.split(appsToTest, ",");
- } else {
- apps = new String[JmeterTestPlanTask.ClientApp.values().length];
- int ctr = 0;
- for (JmeterTestPlanTask.ClientApp app : JmeterTestPlanTask.ClientApp.values())
- apps[ctr++] = app.getId();
- }
-
- for (String app : apps) {
-
- int interval = Integer.valueOf(amsJmeterProperties.getProperty("get-interval"));
- String intervalString = amsJmeterProperties.getProperty(app + "-get-interval");
- if (intervalString != null && !intervalString.isEmpty()) {
- interval = Integer.valueOf(intervalString);
- }
- appGetMetrics.add(new AppGetMetric(readMetricsFromFile(app), interval, app));
- }
-
- return appGetMetrics;
- }
-
- public void runTest(int numInstances) {
-
- int appRefreshRate = Integer.valueOf(amsJmeterProperties.getProperty("app-refresh-rate"));
- for (int i = 0; i < numInstances; i++) {
- ScheduledFuture future = scheduledExecutorService.scheduleAtFixedRate(new JmeterTestPlanTask(appGetMetrics,
- amsJmeterProperties), 0, appRefreshRate, TimeUnit.MILLISECONDS);
- }
- }
-
- /**
- * Sample Usage:
- * java -cp "lib/*":ambari-metrics-timelineservice-2.1.1.0.jar org.apache.hadoop.yarn.server.applicationhistoryservice
- * .metrics.loadsimulator.jmeter.AMSJMeterLoadTest
- * -t UI -p ambari-metrics-timelineservice/src/main/resources/jmeter/ams-jmeter.properties
- */
- public static void main(String[] args) {
- Map<String, String> mapArgs = parseArgs(args);
- String testType = mapArgs.get("type");
- String amsJmeterPropertiesFile = mapArgs.get("amsJmeterPropertiesFile");
- new AMSJMeterLoadTest(testType, amsJmeterPropertiesFile);
- }
-
- private static Map<String, String> parseArgs(String[] args) {
- Map<String, String> mapProps = new HashMap<String, String>();
- if (args.length == 0) {
- printUsage();
- throw new RuntimeException("Unexpected argument, See usage message.");
- } else {
- for (int i = 0; i < args.length; i += 2) {
- String arg = args[i];
- if (arg.equals("-t")) {
- mapProps.put("type", args[i + 1]);
- } else if (arg.equals("-p")) {
- mapProps.put("amsJmeterPropertiesFile", args[i + 1]);
- } else {
- printUsage();
- throw new IllegalArgumentException("Unexpected argument, See usage message.");
- }
- }
- }
- return mapProps;
- }
-
- public static void printUsage() {
- System.err.println("Usage: java AMSJmeterLoadTest [OPTIONS]");
- System.err.println("Options: ");
- System.err.println("[-t type (S=>Sink/U=>UI)] [-p amsJmeterPropertiesFile (Optional)]");
- }
-
-}
-
-
http://git-wip-us.apache.org/repos/asf/ambari/blob/05a4b66b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/jmetertest/AppGetMetric.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/jmetertest/AppGetMetric.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/jmetertest/AppGetMetric.java
deleted file mode 100644
index 727a1c7..0000000
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/jmetertest/AppGetMetric.java
+++ /dev/null
@@ -1,57 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p/>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p/>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.loadsimulator.jmetertest;
-
-import java.util.List;
-
-public class AppGetMetric {
-
- private String app;
- private int interval;
- private List<GetMetricRequestInfo> requests;
-
- public AppGetMetric(List<GetMetricRequestInfo> requests, int interval, String app) {
- this.setMetricRequests(requests);
- this.setInterval(interval);
- this.setApp(app);
- }
-
- public List<GetMetricRequestInfo> getMetricRequests() {
- return requests;
- }
-
- public void setMetricRequests(List<GetMetricRequestInfo> requests) {
- this.requests = requests;
- }
-
- public int getInterval() {
- return interval;
- }
-
- public void setInterval(int interval) {
- this.interval = interval;
- }
-
- public String getApp() {
- return app;
- }
-
- public void setApp(String app) {
- this.app = app;
- }
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/05a4b66b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/jmetertest/GetMetricRequestInfo.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/jmetertest/GetMetricRequestInfo.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/jmetertest/GetMetricRequestInfo.java
deleted file mode 100644
index 26c5025..0000000
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/jmetertest/GetMetricRequestInfo.java
+++ /dev/null
@@ -1,61 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.loadsimulator.jmetertest;
-
-import org.apache.commons.lang.StringUtils;
-
-import java.util.List;
-
-
-public class GetMetricRequestInfo {
-
- private String metricStringPayload;
- private boolean needsTimestamps;
- private boolean needsHost;
-
- public GetMetricRequestInfo(List<String> metrics, boolean needsTimestamps, boolean needsHost) {
-
- this.setMetricStringPayload(StringUtils.join(metrics, ","));
- this.setNeedsTimestamps(needsTimestamps);
- this.setNeedsHost(needsHost);
- }
-
- public String getMetricStringPayload() {
- return metricStringPayload;
- }
-
- public void setMetricStringPayload(String metricStringPayload) {
- this.metricStringPayload = metricStringPayload;
- }
-
- public boolean needsTimestamps() {
- return needsTimestamps;
- }
-
- public void setNeedsTimestamps(boolean needsTimestamps) {
- this.needsTimestamps = needsTimestamps;
- }
-
- public boolean needsHost() {
- return needsHost;
- }
-
- public void setNeedsHost(boolean needsHost) {
- this.needsHost = needsHost;
- }
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/05a4b66b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/jmetertest/JmeterTestPlanTask.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/jmetertest/JmeterTestPlanTask.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/jmetertest/JmeterTestPlanTask.java
deleted file mode 100644
index f7e27b1..0000000
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/jmetertest/JmeterTestPlanTask.java
+++ /dev/null
@@ -1,269 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.loadsimulator.jmetertest;
-
-import org.apache.commons.io.IOUtils;
-import org.apache.jmeter.control.LoopController;
-import org.apache.jmeter.engine.StandardJMeterEngine;
-import org.apache.jmeter.protocol.http.sampler.HTTPSampler;
-import org.apache.jmeter.protocol.http.util.HTTPConstants;
-import org.apache.jmeter.reporters.ResultCollector;
-import org.apache.jmeter.reporters.Summariser;
-import org.apache.jmeter.testelement.TestElement;
-import org.apache.jmeter.testelement.TestPlan;
-import org.apache.jmeter.threads.JMeterContextService;
-import org.apache.jmeter.threads.ThreadGroup;
-import org.apache.jmeter.timers.ConstantTimer;
-import org.apache.jmeter.util.JMeterUtils;
-import org.apache.jorphan.collections.HashTree;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.*;
-import java.lang.reflect.Field;
-import java.util.*;
-
-public class JmeterTestPlanTask implements Runnable {
-
- private static StandardJMeterEngine jmeterEngine = null;
- private final static Logger LOG = LoggerFactory.getLogger(JmeterTestPlanTask.class);
- private List<AppGetMetric> appGetMetrics;
- private Properties amsJmeterProperties;
- private HashTree amsTestPlanTree;
- private TestPlan amsTestPlan;
- private static final String JMETER_HOME = "loadsimulator";
- private static final String JMETER_PROPERTIES_FILE = JMETER_HOME + "/jmeter.properties";
- private static final String SAVESERVICE_PROPERTIES_FILE = JMETER_HOME + "/saveservice.properties";
-
- public enum ClientApp {
- HOST("HOST"),
- NAMENODE("NAMENODE"),
- HBASE("HBASE"),
- NIMBUS("NIMBUS"),
- KAFKA_BROKER("KAFKA_BROKER"),
- FLUME_HANDLER("FLUME_HANDLER"),
- AMS_HBASE("AMS-HBASE"),
- NODEMANAGER("NODEMANAGER"),
- RESOURCEMANAGER("RESOURCEMANAGER"),
- DATANODE("DATANODE");
-
- private String id;
-
- private ClientApp(String id) {
- this.id = id;
- }
-
- public String getId() {
- return id;
- }
- }
-
- public JmeterTestPlanTask(List<AppGetMetric> appGetMetrics, Properties amsJmeterProperties) {
- this.appGetMetrics = appGetMetrics;
- this.amsJmeterProperties = amsJmeterProperties;
- amsTestPlanTree = new HashTree();
- amsTestPlan = new TestPlan("AMS JMeter Load Test plan");
- System.out.println("Starting AMS Jmeter load testing");
- }
-
- public void run() {
- if (jmeterEngine != null) {
-
- Object[] threadGroups = amsTestPlanTree.getArray(amsTestPlan);
- for (Object threadGroupObj : threadGroups) {
- if (threadGroupObj instanceof ThreadGroup) {
- ThreadGroup threadGroup = (ThreadGroup) threadGroupObj;
- threadGroup.stop();
- }
- }
- amsTestPlanTree.clear();
- jmeterEngine.askThreadsToStop();
- jmeterEngine.stopTest();
- JMeterContextService.endTest();
- }
-
- //Start the new test plan for the new app.
- try {
- //Initialize Jmeter essentials
- jmeterEngine = new StandardJMeterEngine();
- JMeterContextService.getContext().setEngine(jmeterEngine);
-
- //Workaround to supply JMeterUtils with jmeter.prooperties from JAR.
- JMeterUtils.setJMeterHome("");
- Field f = new JMeterUtils().getClass().getDeclaredField("appProperties");
- f.setAccessible(true);
- f.set(null, AMSJMeterLoadTest.readProperties(JMETER_PROPERTIES_FILE));
-
- //Copy saveservices.properties file to tmp dir for JMeter to consume.
- InputStream inputStream = ClassLoader.getSystemResourceAsStream(SAVESERVICE_PROPERTIES_FILE);
- if (inputStream == null) {
- inputStream = new FileInputStream(SAVESERVICE_PROPERTIES_FILE);
- }
- String tmpDir = System.getProperty("java.io.tmpdir");
- OutputStream outputStream = new FileOutputStream(tmpDir + "/saveservice.properties");
- IOUtils.copy(inputStream, outputStream);
- outputStream.close();
- JMeterUtils.setProperty("saveservice_properties", tmpDir + "/saveservice.properties");
-
- //Initialize Test plan
- amsTestPlan.setProperty(TestElement.TEST_CLASS, TestPlan.class.getName());
- amsTestPlanTree.add("AMS Test plan", amsTestPlan);
-
- //Choose a random APP to run the perform GET metrics request.
- int currentAppIndex = new Random().nextInt(appGetMetrics.size());
-
- //Create ThreadGroup for the App
- createThreadGroupHashTree(currentAppIndex, amsJmeterProperties, amsTestPlanTree, amsTestPlan);
-
- //Geneates the JMX file that you can use through the GUI mode.
- //SaveService.saveTree(amsTestPlanTree, new FileOutputStream(JMETER_HOME + "/" + "amsTestPlan.jmx"));
-
- //Summarizer output to get test progress in stdout like.
- Summariser summariser = null;
- String summariserName = JMeterUtils.getPropDefault("summariser.name", "summary");
- if (summariserName.length() > 0) {
- summariser = new Summariser(summariserName);
- }
-
- //Store execution results into a .jtl file
- String jmeterLogFile = tmpDir + "/amsJmeterTestResults.jtl";
- ResultCollector resultCollector = new ResultCollector(summariser);
- resultCollector.setFilename(jmeterLogFile);
- amsTestPlanTree.add(amsTestPlanTree.getArray()[0], resultCollector);
- jmeterEngine.configure(amsTestPlanTree);
- jmeterEngine.run();
-
- LOG.info("AMS Jmeter Test started up successfully");
-
- } catch (Exception ioEx) {
- amsTestPlanTree.clear();
- jmeterEngine.askThreadsToStop();
- jmeterEngine.stopTest();
- JMeterContextService.endTest();
- LOG.error("Error occurred while running AMS load test : " + ioEx.getMessage());
- ioEx.printStackTrace();
- }
- }
-
- private ConstantTimer createConstantTimer(int delay) {
- ConstantTimer timer = new ConstantTimer();
- timer.setDelay("" + delay);
- return timer;
- }
-
- private Map<String, String> getAppSpecificParameters(String app, GetMetricRequestInfo request, Properties amsJmeterProperties) {
-
- Map<String, String> parametersMap = new HashMap<String, String>();
- String hostPrefix = amsJmeterProperties.getProperty("host-prefix");
- String hostSuffix = amsJmeterProperties.getProperty("host-suffix");
- int minHostIndex = Integer.valueOf(amsJmeterProperties.getProperty("min-host-index"));
- int numHosts = Integer.valueOf(amsJmeterProperties.getProperty("num-hosts"));
-
- parametersMap.put("appId", app);
-
- if (request.needsTimestamps()) {
- long currentTime = System.currentTimeMillis();
- long oneHourBack = currentTime - 3600 * 1000;
- parametersMap.put("startTime", String.valueOf(oneHourBack));
- parametersMap.put("endTime", String.valueOf(currentTime));
- }
-
- if (request.needsHost()) {
- if (ClientApp.AMS_HBASE.getId().equals(app)) {
- parametersMap.put("hostname", amsJmeterProperties.getProperty("ams-host"));
- } else if (ClientApp.HOST.getId().equals(app) || ClientApp.NODEMANAGER.getId().equals(app)) {
- int randomHost = minHostIndex + new Random().nextInt(numHosts);
- parametersMap.put("hostname", hostPrefix + randomHost + hostSuffix);
- } else {
- parametersMap.put("hostname", hostPrefix + amsJmeterProperties.getProperty(app + "-host") + hostSuffix);
- }
- }
- parametersMap.put("metricNames", request.getMetricStringPayload());
- return parametersMap;
- }
-
- private void createThreadGroupHashTree(int appIndex, Properties amsJmeterProperties, HashTree amsTestPlanTree, TestPlan amsTestPlan) {
-
- AppGetMetric appGetMetric = appGetMetrics.get(appIndex);
- String app = appGetMetric.getApp();
- int interval = appGetMetric.getInterval();
-
- //Read and validate AMS information.
- String[] amsHostPort = amsJmeterProperties.getProperty("ams-host-port").split(":");
- String amsHost = amsHostPort[0];
- String amsPath = amsJmeterProperties.getProperty("ams-path");
- int amsPort = Integer.valueOf(amsHostPort[1]);
- int numLoops = Integer.valueOf(amsJmeterProperties.getProperty("num-get-calls-per-app"));
-
- LoopController loopController = createLoopController(app + " GET loop controller", numLoops, false);
- for (GetMetricRequestInfo request : appGetMetric.getMetricRequests()) {
-
- ThreadGroup threadGroup = createThreadGroup(app + " GET threadGroup", 1, 0, loopController);
-
- HashTree threadGroupHashTree = amsTestPlanTree.add(amsTestPlan, threadGroup);
- Map<String, String> parametersMap = getAppSpecificParameters(app, request, amsJmeterProperties);
-
- HTTPSampler sampler = createGetSampler("GET " + app + " metrics", amsHost, amsPort, amsPath, null, parametersMap);
-
- if (numLoops > 1) {
- threadGroupHashTree.add(createConstantTimer(interval));
- }
-
- threadGroupHashTree.add(sampler);
- }
- }
-
- private HTTPSampler createGetSampler(String name, String domain, int port, String path, String encoding, Map<String, String> parameters) {
-
- HTTPSampler sampler = new HTTPSampler();
- sampler.setDomain(domain);
- sampler.setPort(port);
- sampler.setPath(path);
- sampler.setMethod(HTTPConstants.GET);
-
- if (encoding != null)
- sampler.setContentEncoding(encoding);
-
- for (Map.Entry<String, String> entry : parameters.entrySet()) {
- sampler.addArgument(entry.getKey(), entry.getValue());
- }
- sampler.setName(name);
- return sampler;
- }
-
- private LoopController createLoopController(String name, int numLoops, boolean continueForever) {
- LoopController loopController = new LoopController();
- loopController.setLoops(numLoops);
- loopController.setProperty(TestElement.TEST_CLASS, LoopController.class.getName());
- loopController.initialize();
- loopController.setContinueForever(continueForever);
- loopController.setName(name);
- return loopController;
- }
-
- private ThreadGroup createThreadGroup(String name, int numThreads, int rampUp, LoopController loopController) {
- ThreadGroup threadGroup = new ThreadGroup();
- threadGroup.setName(name);
- threadGroup.setNumThreads(numThreads);
- threadGroup.setRampUp(rampUp);
- threadGroup.setSamplerController(loopController);
- threadGroup.setProperty(TestElement.TEST_CLASS, ThreadGroup.class.getName());
- return threadGroup;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/05a4b66b/ambari-metrics/ambari-metrics-timelineservice/src/main/resources/loadsimulator/README
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/resources/loadsimulator/README b/ambari-metrics/ambari-metrics-timelineservice/src/main/resources/loadsimulator/README
deleted file mode 100644
index 39e5365..0000000
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/resources/loadsimulator/README
+++ /dev/null
@@ -1,65 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
- WORKING OF THE AMS LOAD SIMULATOR
- The AMS load simulator is designed to perform load testing on a live AMS instance by performing the role of either a
- sink or an Ambari UI instance based on how it is being invoked.
-
- > When it acts as a host sink, it makes calls to the AMS to PUT metrics for all the services for a defined interval.
- The simulator can also be used to start up "N" such host instances where each instance has a number of Sinks that PUT
- metrics.
- > When the load simulator is invoked as a UI instance, it makes GET metrics calls to the AMS in defined
- intervals for all the services. The rate of the GET metrics call and the list of metrics requested has been designed
- to closely match an actual Ambari UI instance. Apache JMeter API has been used to design the GET calls made to the
- AMS.
-
- The load simulator uses a properties file (ams-jmeter.properties) file to configure the test run. It is part of the
- JAR in the same folder as this README file. It can also be supplied as a command line argument to the test using the
- "-p" option. Other properties files like jmeter.properties and saveservice.properties contain JMeter internal
- properties and need not be modified.
-
-
- INSTRUCTIONS TO RUN THE SIMULATOR
- 1. Modify the ams-jmeter.properties to point to your AMS host. Change the properties "num-hosts" based on how many hosts
- need to be simulated for sinks. The GET Metric section of the properties is used for fine tuning GET call interval
- for every APP type.
- 2. Build the ambari-metrics-timelineservice jar.
- 3. Invoke the test using the command as follows.
-
- java -cp lib/*:ambari-metrics-timelineservice-<version>.jar org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.loadsimulator.jmeter.AMSJMeterLoadTest -t <S/U>
-
- lib/* -> All the dependency JARs generated for the ambari-metrics-timelineservice JAR.
- -t option => S-Sink simulator or U-UI simulator
- You can use the -p <location of ams-jmeter.properties> option to pass in your own properties file.
-
- 4. Test results will be found at <TMP_DIR>/amsTestResults.jtl.
- 5. Open the amsJmeterGrpah.jmx file through a JMeter GUI instance and supply the results (amsTestResults.jtl) file as
- input to the Graph to be drawn.
-
- TESTING ON GCE
-
- 1. Copy the JAR, libs, optional ams-jmeter.properties file to all the machines on which the test needs to be run.
- 2. Sink simulation for num-hosts = N.
- Start the test with -t S on 1 machine with property "create-master=true".
- Start the test with -t S on N-1 machines with property "create-master=false"
- 3. UI simulation
- Start the test with -t U on 1 or more machines.
- 4. To stop the test after you have sufficient load testing done, the following command should be run on all machines.
- ps axf | grep jmeter | grep -v grep | awk '{print "kill -9 " $1}' | sh
- 5. Copy over the results file to a location with JMeter downloaded. Open the amsJmeterGraph.jmx on jmeter and browse to
- open the results file as input to the graph.
http://git-wip-us.apache.org/repos/asf/ambari/blob/05a4b66b/ambari-metrics/ambari-metrics-timelineservice/src/main/resources/loadsimulator/ams-jmeter.properties
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/resources/loadsimulator/ams-jmeter.properties b/ambari-metrics/ambari-metrics-timelineservice/src/main/resources/loadsimulator/ams-jmeter.properties
deleted file mode 100644
index 3353d43..0000000
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/resources/loadsimulator/ams-jmeter.properties
+++ /dev/null
@@ -1,56 +0,0 @@
-# Copyright 2011 The Apache Software Foundation
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-#AMS information
-ams-host-port=104.196.94.27:6188
-ams-path=/ws/v1/timeline/metrics
-
-# Host Information, using the host-prefix, num-hosts and <MASTER>-host
-# Hosts in this case will be TestHost0, TestHost1
-# For example NAMENODE, NIMBUS,HBASE MASTER,RESOURCEMANAGER host will be TestHost0
-host-prefix=TestHost
-host-suffix=
-min-host-index=0
-num-hosts=2
-NAMENODE-host=0
-NIMBUS-host=0
-HBASE-host=0
-RESOURCEMANAGER-host=0
-
-# PUT Metric / Sinks config
-collection-interval=1000
-send-interval=3000
-create-master=true
-
-# GET Metric / Client Apps config
-num-ui-instances=1
-apps-to-test=
-app-refresh-rate=8000
-num-get-calls-per-app=3
-get-interval=3000
-HOST-get-interval=3000
-NAMENODE-get-interval=2000
-HBASE-get-interval=3000
-NIMBUS-get-interval=2000
-AMS-HBASE-get-interval=2000
-FLUME_HANDLER-get-interval=2000
-NODEMANAGER-get-interval=2000
-KAFKA_BROKER-get-interval=2000
-DATANODE-get-interval=3000
-RESOURCEMANAGER-get-interval=3000
-
http://git-wip-us.apache.org/repos/asf/ambari/blob/05a4b66b/ambari-metrics/ambari-metrics-timelineservice/src/main/resources/loadsimulator/amsJmeterGraph.jmx
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/resources/loadsimulator/amsJmeterGraph.jmx b/ambari-metrics/ambari-metrics-timelineservice/src/main/resources/loadsimulator/amsJmeterGraph.jmx
deleted file mode 100644
index 06d6360..0000000
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/resources/loadsimulator/amsJmeterGraph.jmx
+++ /dev/null
@@ -1,104 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one
- ~ or more contributor license agreements. See the NOTICE file
- ~ distributed with this work for additional information
- ~ regarding copyright ownership. The ASF licenses this file
- ~ to you under the Apache License, Version 2.0 (the
- ~ "License"); you may not use this file except in compliance
- ~ with the License. You may obtain a copy of the License at
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing, software
- ~ distributed under the License is distributed on an "AS IS" BASIS,
- ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- ~ See the License for the specific language governing permissions and
- ~ limitations under the License.
- -->
-<jmeterTestPlan version="1.2" properties="2.8" jmeter="2.13 r1665067">
- <hashTree>
- <TestPlan guiclass="TestPlanGui" testclass="TestPlan" testname="Test Plan" enabled="true">
- <stringProp name="TestPlan.comments"></stringProp>
- <boolProp name="TestPlan.functional_mode">false</boolProp>
- <boolProp name="TestPlan.serialize_threadgroups">false</boolProp>
- <elementProp name="TestPlan.user_defined_variables" elementType="Arguments" guiclass="ArgumentsPanel" testclass="Arguments" testname="User Defined Variables" enabled="true">
- <collectionProp name="Arguments.arguments"/>
- </elementProp>
- <stringProp name="TestPlan.user_define_classpath"></stringProp>
- </TestPlan>
- <hashTree>
- <ResultCollector guiclass="GraphVisualizer" testclass="ResultCollector" testname="Latency Graph" enabled="true">
- <boolProp name="ResultCollector.error_logging">false</boolProp>
- <objProp>
- <name>saveConfig</name>
- <value class="SampleSaveConfiguration">
- <time>false</time>
- <latency>false</latency>
- <timestamp>true</timestamp>
- <success>true</success>
- <label>true</label>
- <code>true</code>
- <message>true</message>
- <threadName>false</threadName>
- <dataType>true</dataType>
- <encoding>false</encoding>
- <assertions>false</assertions>
- <subresults>false</subresults>
- <responseData>true</responseData>
- <samplerData>true</samplerData>
- <xml>true</xml>
- <fieldNames>true</fieldNames>
- <responseHeaders>true</responseHeaders>
- <requestHeaders>true</requestHeaders>
- <responseDataOnError>true</responseDataOnError>
- <saveAssertionResultsFailureMessage>false</saveAssertionResultsFailureMessage>
- <assertionsResultsToSave>0</assertionsResultsToSave>
- <url>true</url>
- <hostname>true</hostname>
- <sampleCount>true</sampleCount>
- </value>
- </objProp>
- <stringProp name="filename">/tmp/amsJmeterTestResults.jtl</stringProp>
- </ResultCollector>
- <hashTree/>
- <ResultCollector guiclass="RespTimeGraphVisualizer" testclass="ResultCollector" testname="Response Time Graph" enabled="true">
- <boolProp name="ResultCollector.error_logging">false</boolProp>
- <objProp>
- <name>saveConfig</name>
- <value class="SampleSaveConfiguration">
- <time>false</time>
- <latency>false</latency>
- <timestamp>true</timestamp>
- <success>true</success>
- <label>true</label>
- <code>true</code>
- <message>true</message>
- <threadName>false</threadName>
- <dataType>true</dataType>
- <encoding>false</encoding>
- <assertions>false</assertions>
- <subresults>false</subresults>
- <responseData>true</responseData>
- <samplerData>true</samplerData>
- <xml>true</xml>
- <fieldNames>true</fieldNames>
- <responseHeaders>true</responseHeaders>
- <requestHeaders>true</requestHeaders>
- <responseDataOnError>true</responseDataOnError>
- <saveAssertionResultsFailureMessage>false</saveAssertionResultsFailureMessage>
- <assertionsResultsToSave>0</assertionsResultsToSave>
- <url>true</url>
- <hostname>true</hostname>
- <sampleCount>true</sampleCount>
- </value>
- </objProp>
- <stringProp name="filename">/tmp/amsJmeterTestResults.jtl</stringProp>
- <stringProp name="RespTimeGraph.interval">15000</stringProp>
- <intProp name="RespTimeGraph.linestrockwidth">2</intProp>
- <intProp name="RespTimeGraph.lineshapepoint">4</intProp>
- </ResultCollector>
- <hashTree/>
- </hashTree>
- </hashTree>
-</jmeterTestPlan>