You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by vi...@apache.org on 2017/11/04 01:31:27 UTC
[22/50] [abbrv] hadoop git commit: HADOOP-15013. Fix
ResourceEstimator findbugs issues. (asuresh)
HADOOP-15013. Fix ResourceEstimator findbugs issues. (asuresh)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/53c0fb7e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/53c0fb7e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/53c0fb7e
Branch: refs/heads/HDFS-9806
Commit: 53c0fb7efebfac4a79f5cce2dd42cf00411d51e7
Parents: e6ec020
Author: Arun Suresh <as...@apache.org>
Authored: Thu Nov 2 17:14:07 2017 -0700
Committer: Arun Suresh <as...@apache.org>
Committed: Thu Nov 2 17:15:33 2017 -0700
----------------------------------------------------------------------
.../service/ResourceEstimatorService.java | 85 +++++++++-----------
.../translator/impl/BaseLogParser.java | 5 +-
.../translator/impl/LogParserUtil.java | 11 ++-
.../service/TestResourceEstimatorService.java | 2 -
4 files changed, 53 insertions(+), 50 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/53c0fb7e/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/service/ResourceEstimatorService.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/service/ResourceEstimatorService.java b/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/service/ResourceEstimatorService.java
index 933332e..0e0e094 100644
--- a/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/service/ResourceEstimatorService.java
+++ b/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/service/ResourceEstimatorService.java
@@ -65,51 +65,49 @@ import com.google.inject.Singleton;
@Singleton @Path("/resourceestimator") public class ResourceEstimatorService {
private static final Logger LOGGER =
LoggerFactory.getLogger(ResourceEstimatorService.class);
- private static SkylineStore skylineStore;
- private static Solver solver;
- private static LogParser logParser;
- private static LogParserUtil logParserUtil = new LogParserUtil();
- private static Configuration config;
- private static Gson gson;
- private static Type rleType;
- private static Type skylineStoreType;
+ private final SkylineStore skylineStore;
+ private final Solver solver;
+ private final LogParser logParser;
+ private final LogParserUtil logParserUtil = new LogParserUtil();
+ private final Configuration config;
+ private final Gson gson;
+ private final Type rleType;
+ private final Type skylineStoreType;
public ResourceEstimatorService() throws ResourceEstimatorException {
- if (skylineStore == null) {
- try {
- config = new Configuration();
- config.addResource(ResourceEstimatorConfiguration.CONFIG_FILE);
- skylineStore = ResourceEstimatorUtil.createProviderInstance(config,
- ResourceEstimatorConfiguration.SKYLINESTORE_PROVIDER,
- ResourceEstimatorConfiguration.DEFAULT_SKYLINESTORE_PROVIDER,
- SkylineStore.class);
- logParser = ResourceEstimatorUtil.createProviderInstance(config,
- ResourceEstimatorConfiguration.TRANSLATOR_PROVIDER,
- ResourceEstimatorConfiguration.DEFAULT_TRANSLATOR_PROVIDER,
- LogParser.class);
- logParser.init(config, skylineStore);
- logParserUtil.setLogParser(logParser);
- solver = ResourceEstimatorUtil.createProviderInstance(config,
- ResourceEstimatorConfiguration.SOLVER_PROVIDER,
- ResourceEstimatorConfiguration.DEFAULT_SOLVER_PROVIDER,
- Solver.class);
- solver.init(config, skylineStore);
- } catch (Exception ex) {
- LOGGER
- .error("Server initialization failed due to: {}", ex.getMessage());
- throw new ResourceEstimatorException(ex.getMessage(), ex);
- }
- gson = new GsonBuilder()
- .registerTypeAdapter(Resource.class, new ResourceSerDe())
- .registerTypeAdapter(RLESparseResourceAllocation.class,
- new RLESparseResourceAllocationSerDe())
- .enableComplexMapKeySerialization().create();
- rleType = new TypeToken<RLESparseResourceAllocation>() {
- }.getType();
- skylineStoreType =
- new TypeToken<Map<RecurrenceId, List<ResourceSkyline>>>() {
- }.getType();
+ try {
+ config = new Configuration();
+ config.addResource(ResourceEstimatorConfiguration.CONFIG_FILE);
+ skylineStore = ResourceEstimatorUtil.createProviderInstance(config,
+ ResourceEstimatorConfiguration.SKYLINESTORE_PROVIDER,
+ ResourceEstimatorConfiguration.DEFAULT_SKYLINESTORE_PROVIDER,
+ SkylineStore.class);
+ logParser = ResourceEstimatorUtil.createProviderInstance(config,
+ ResourceEstimatorConfiguration.TRANSLATOR_PROVIDER,
+ ResourceEstimatorConfiguration.DEFAULT_TRANSLATOR_PROVIDER,
+ LogParser.class);
+ logParser.init(config, skylineStore);
+ logParserUtil.setLogParser(logParser);
+ solver = ResourceEstimatorUtil.createProviderInstance(config,
+ ResourceEstimatorConfiguration.SOLVER_PROVIDER,
+ ResourceEstimatorConfiguration.DEFAULT_SOLVER_PROVIDER,
+ Solver.class);
+ solver.init(config, skylineStore);
+ } catch (Exception ex) {
+ LOGGER
+ .error("Server initialization failed due to: {}", ex.getMessage());
+ throw new ResourceEstimatorException(ex.getMessage(), ex);
}
+ gson = new GsonBuilder()
+ .registerTypeAdapter(Resource.class, new ResourceSerDe())
+ .registerTypeAdapter(RLESparseResourceAllocation.class,
+ new RLESparseResourceAllocationSerDe())
+ .enableComplexMapKeySerialization().create();
+ rleType = new TypeToken<RLESparseResourceAllocation>() {
+ }.getType();
+ skylineStoreType =
+ new TypeToken<Map<RecurrenceId, List<ResourceSkyline>>>() {
+ }.getType();
}
/**
@@ -192,9 +190,6 @@ import com.google.inject.Singleton;
LOGGER
.debug("Query the skyline store for recurrenceId: {}." + recurrenceId);
- recurrenceId = new RecurrenceId("*", "*");
- jobHistory = skylineStore.getHistory(recurrenceId);
-
return skyline;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/53c0fb7e/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/translator/impl/BaseLogParser.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/translator/impl/BaseLogParser.java b/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/translator/impl/BaseLogParser.java
index 50d911f..afafd55 100644
--- a/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/translator/impl/BaseLogParser.java
+++ b/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/translator/impl/BaseLogParser.java
@@ -24,11 +24,13 @@ import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
+import java.nio.charset.StandardCharsets;
import java.text.ParseException;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import org.apache.commons.lang.CharSet;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.resourceestimator.common.api.RecurrenceId;
import org.apache.hadoop.resourceestimator.common.api.ResourceSkyline;
@@ -101,7 +103,8 @@ public class BaseLogParser implements LogParser {
new HashMap<>();
final Map<String, JobMetaData> jobMetas =
new HashMap<String, JobMetaData>();
- final BufferedReader bf = new BufferedReader(new InputStreamReader(logs));
+ final BufferedReader bf = new BufferedReader(
+ new InputStreamReader(logs, StandardCharsets.UTF_8));
String line = null;
while ((line = bf.readLine()) != null) {
try {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/53c0fb7e/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/translator/impl/LogParserUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/translator/impl/LogParserUtil.java b/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/translator/impl/LogParserUtil.java
index 35da799..4ca1fff 100644
--- a/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/translator/impl/LogParserUtil.java
+++ b/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/translator/impl/LogParserUtil.java
@@ -91,7 +91,14 @@ public class LogParserUtil {
throw new ResourceEstimatorException("The log parser is not initialized,"
+ " please try again after initializing.");
}
- InputStream inputStream = new FileInputStream(logFile);
- logParser.parseStream(inputStream);
+ InputStream inputStream = null;
+ try {
+ inputStream = new FileInputStream(logFile);
+ logParser.parseStream(inputStream);
+ } finally {
+ if (inputStream != null) {
+ inputStream.close();
+ }
+ }
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/53c0fb7e/hadoop-tools/hadoop-resourceestimator/src/test/java/org/apache/hadoop/resourceestimator/service/TestResourceEstimatorService.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-resourceestimator/src/test/java/org/apache/hadoop/resourceestimator/service/TestResourceEstimatorService.java b/hadoop-tools/hadoop-resourceestimator/src/test/java/org/apache/hadoop/resourceestimator/service/TestResourceEstimatorService.java
index d4dde7e..91a486e 100644
--- a/hadoop-tools/hadoop-resourceestimator/src/test/java/org/apache/hadoop/resourceestimator/service/TestResourceEstimatorService.java
+++ b/hadoop-tools/hadoop-resourceestimator/src/test/java/org/apache/hadoop/resourceestimator/service/TestResourceEstimatorService.java
@@ -54,8 +54,6 @@ import com.sun.jersey.test.framework.WebAppDescriptor;
* Test ResourceEstimatorService.
*/
public class TestResourceEstimatorService extends JerseyTest {
- private static final Logger LOGGER =
- LoggerFactory.getLogger(TestResourceEstimatorService.class);
private final String parseLogCommand = "resourceestimator/translator/"
+ "src/test/resources/resourceEstimatorService.txt";
private final String getHistorySkylineCommand =
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org