You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by aa...@apache.org on 2017/10/26 06:34:36 UTC
[5/5] hadoop git commit: Revert "HADOOP-14840. Tool to estimate
resource requirements of an application pipeline based on prior executions.
(Rui Li via Subru)."
Revert "HADOOP-14840. Tool to estimate resource requirements of an application pipeline based on prior executions. (Rui Li via Subru)."
This reverts commit da7f989d86d28ee6f4a7ac410fc8a4a163e11012.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/13f1a7e0
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/13f1a7e0
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/13f1a7e0
Branch: refs/heads/branch-2
Commit: 13f1a7e0edf05e18963b1b00999dc9f320c8cd53
Parents: da7f989
Author: Akira Ajisaka <aa...@apache.org>
Authored: Thu Oct 26 15:33:53 2017 +0900
Committer: Akira Ajisaka <aa...@apache.org>
Committed: Thu Oct 26 15:33:53 2017 +0900
----------------------------------------------------------------------
LICENSE.txt | 1 -
.../assemblies/hadoop-resourceestimator.xml | 42 --
.../main/resources/assemblies/hadoop-tools.xml | 11 -
hadoop-project/pom.xml | 5 -
hadoop-project/src/site/site.xml | 1 -
hadoop-tools/hadoop-resourceestimator/README.md | 19 -
hadoop-tools/hadoop-resourceestimator/pom.xml | 175 -------
.../src/config/checkstyle.xml | 50 --
.../src/main/bin/estimator.cmd | 52 ---
.../src/main/bin/estimator.sh | 71 ---
.../src/main/bin/start-estimator.cmd | 37 --
.../src/main/bin/start-estimator.sh | 42 --
.../src/main/bin/stop-estimator.cmd | 37 --
.../src/main/bin/stop-estimator.sh | 42 --
.../src/main/conf/resourceestimator-config.xml | 85 ----
.../src/main/data/resourceEstimatorService.txt | 2 -
.../common/api/RecurrenceId.java | 95 ----
.../common/api/ResourceSkyline.java | 211 ---------
.../common/api/package-info.java | 23 -
.../config/ResourceEstimatorConfiguration.java | 125 -----
.../common/config/ResourceEstimatorUtil.java | 81 ----
.../common/config/package-info.java | 23 -
.../exception/ResourceEstimatorException.java | 35 --
.../common/exception/package-info.java | 23 -
.../RLESparseResourceAllocationSerDe.java | 77 ---
.../common/serialization/ResourceSerDe.java | 61 ---
.../common/serialization/package-info.java | 24 -
.../service/ResourceEstimatorServer.java | 146 ------
.../service/ResourceEstimatorService.java | 238 ----------
.../resourceestimator/service/ShutdownHook.java | 45 --
.../resourceestimator/service/package-info.java | 23 -
.../skylinestore/api/HistorySkylineStore.java | 99 ----
.../api/PredictionSkylineStore.java | 60 ---
.../skylinestore/api/SkylineStore.java | 30 --
.../skylinestore/api/package-info.java | 23 -
.../DuplicateRecurrenceIdException.java | 33 --
.../EmptyResourceSkylineException.java | 33 --
.../exceptions/NullPipelineIdException.java | 32 --
...ullRLESparseResourceAllocationException.java | 33 --
.../exceptions/NullRecurrenceIdException.java | 32 --
.../NullResourceSkylineException.java | 32 --
.../RecurrenceIdNotFoundException.java | 33 --
.../exceptions/SkylineStoreException.java | 33 --
.../skylinestore/exceptions/package-info.java | 24 -
.../skylinestore/impl/InMemoryStore.java | 256 ----------
.../skylinestore/impl/package-info.java | 23 -
.../validator/SkylineStoreValidator.java | 118 -----
.../skylinestore/validator/package-info.java | 23 -
.../resourceestimator/solver/api/Solver.java | 76 ---
.../solver/api/package-info.java | 23 -
.../exceptions/InvalidInputException.java | 34 --
.../exceptions/InvalidSolverException.java | 34 --
.../solver/exceptions/SolverException.java | 34 --
.../solver/exceptions/package-info.java | 24 -
.../solver/impl/BaseSolver.java | 94 ----
.../resourceestimator/solver/impl/LpSolver.java | 340 --------------
.../solver/impl/package-info.java | 23 -
.../solver/preprocess/SolverPreprocessor.java | 219 ---------
.../solver/preprocess/package-info.java | 23 -
.../translator/api/JobMetaData.java | 163 -------
.../translator/api/LogParser.java | 65 ---
.../translator/api/SingleLineParser.java | 52 ---
.../translator/api/package-info.java | 23 -
.../exceptions/DataFieldNotFoundException.java | 32 --
.../translator/exceptions/package-info.java | 23 -
.../translator/impl/BaseLogParser.java | 125 -----
.../translator/impl/LogParserUtil.java | 97 ----
.../translator/impl/NativeSingleLineParser.java | 120 -----
.../translator/impl/RmSingleLineParser.java | 203 --------
.../translator/impl/package-info.java | 23 -
.../translator/validator/ParserValidator.java | 41 --
.../translator/validator/package-info.java | 23 -
.../webapps/ResourceEstimatorServer/.gitignore | 14 -
.../src/site/markdown/ResourceEstimator.md | 181 --------
.../src/site/resources/css/site.css | 29 --
.../common/api/TestResourceSkyline.java | 128 -----
.../serialization/TestHistorySkylineSerDe.java | 134 ------
.../common/serialization/TestResourceSerDe.java | 64 ---
.../serialization/TestResourceSkylineSerDe.java | 112 -----
.../common/serialization/package-info.java | 24 -
.../service/GuiceServletConfig.java | 42 --
.../service/TestResourceEstimatorService.java | 282 -----------
.../skylinestore/impl/TestInMemoryStore.java | 32 --
.../skylinestore/impl/TestSkylineStore.java | 464 -------------------
.../solver/impl/TestLpSolver.java | 112 -----
.../solver/impl/TestSolver.java | 73 ---
.../translator/api/TestJobMetaData.java | 163 -------
.../translator/impl/TestNativeParser.java | 115 -----
.../translator/impl/TestRmParser.java | 239 ----------
.../src/test/resources/log4j.properties | 25 -
.../src/test/resources/lp/answer.txt | 7 -
.../src/test/resources/lp/tinySample.txt | 2 -
.../src/test/resources/lp/tpch_q12.txt | 13 -
.../test/resources/resourceEstimatorService.txt | 2 -
.../test/resources/resourceestimator-config.xml | 85 ----
.../src/test/resources/trace/invalidLog1.txt | 4 -
.../src/test/resources/trace/invalidLog2.txt | 3 -
.../src/test/resources/trace/invalidLog3.txt | 5 -
.../src/test/resources/trace/invalidLog4.txt | 5 -
.../src/test/resources/trace/invalidLog5.txt | 7 -
.../src/test/resources/trace/invalidLog6.txt | 5 -
.../src/test/resources/trace/invalidLog7.txt | 5 -
.../src/test/resources/trace/invalidLog8.txt | 6 -
.../src/test/resources/trace/invalidLog9.txt | 6 -
.../src/test/resources/trace/nativeLog.txt | 1 -
.../src/test/resources/trace/rmLog.txt | 30 --
hadoop-tools/hadoop-tools-dist/pom.xml | 6 -
hadoop-tools/pom.xml | 1 -
108 files changed, 7159 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/13f1a7e0/LICENSE.txt
----------------------------------------------------------------------
diff --git a/LICENSE.txt b/LICENSE.txt
index 09a25d8..ebabcf9 100644
--- a/LICENSE.txt
+++ b/LICENSE.txt
@@ -618,7 +618,6 @@ hadoop-tools/hadoop-sls/src/main/html/js/thirdparty/jquery.js
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/jquery
Apache HBase - Server which contains JQuery minified javascript library version 1.8.3
Microsoft JDBC Driver for SQLServer - version 6.2.1.jre7
-oj! Algorithms - version 43.0
--------------------------------------------------------------------------------
Copyright jQuery Foundation and other contributors, https://jquery.org/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/13f1a7e0/hadoop-assemblies/src/main/resources/assemblies/hadoop-resourceestimator.xml
----------------------------------------------------------------------
diff --git a/hadoop-assemblies/src/main/resources/assemblies/hadoop-resourceestimator.xml b/hadoop-assemblies/src/main/resources/assemblies/hadoop-resourceestimator.xml
deleted file mode 100644
index d1d941d..0000000
--- a/hadoop-assemblies/src/main/resources/assemblies/hadoop-resourceestimator.xml
+++ /dev/null
@@ -1,42 +0,0 @@
-<!--
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.3"
- xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.3 http://maven.apache.org/xsd/assembly-1.1.3.xsd">
- <id>hadoop-resourceestimator</id>
- <formats>
- <format>dir</format>
- </formats>
- <includeBaseDirectory>false</includeBaseDirectory>
-
- <fileSets>
- <fileSet>
- <directory>${basedir}/src/main/bin</directory>
- <outputDirectory>resourceestimator/bin</outputDirectory>
- <fileMode>0755</fileMode>
- </fileSet>
- <fileSet>
- <directory>${basedir}/src/main/conf</directory>
- <outputDirectory>resourceestimator/conf</outputDirectory>
- </fileSet>
- <fileSet>
- <directory>${basedir}/src/main/data</directory>
- <outputDirectory>resourceestimator/data</outputDirectory>
- </fileSet>
- </fileSets>
- </assembly>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/13f1a7e0/hadoop-assemblies/src/main/resources/assemblies/hadoop-tools.xml
----------------------------------------------------------------------
diff --git a/hadoop-assemblies/src/main/resources/assemblies/hadoop-tools.xml b/hadoop-assemblies/src/main/resources/assemblies/hadoop-tools.xml
index e6a9a76..d7c3c95 100644
--- a/hadoop-assemblies/src/main/resources/assemblies/hadoop-tools.xml
+++ b/hadoop-assemblies/src/main/resources/assemblies/hadoop-tools.xml
@@ -118,17 +118,6 @@
<directory>../hadoop-sls/target/hadoop-sls-${project.version}/sls</directory>
<outputDirectory>/share/hadoop/${hadoop.component}/sls</outputDirectory>
</fileSet>
- <fileSet>
- <directory>../hadoop-resourceestimator/target</directory>
- <outputDirectory>/share/hadoop/${hadoop.component}/sources</outputDirectory>
- <includes>
- <include>*-sources.jar</include>
- </includes>
- </fileSet>
- <fileSet>
- <directory>../hadoop-resourceestimator/target/hadoop-resourceestimator-${project.version}/resourceestimator</directory>
- <outputDirectory>/share/hadoop/${hadoop.component}/resourceestimator</outputDirectory>
- </fileSet>
</fileSets>
<dependencySets>
<dependencySet>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/13f1a7e0/hadoop-project/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index ea02765..2c8f4a9 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -618,11 +618,6 @@
<version>3.2</version>
</dependency>
<dependency>
- <groupId>org.ojalgo</groupId>
- <artifactId>ojalgo</artifactId>
- <version>43.0</version>
- </dependency>
- <dependency>
<groupId>com.sun.jersey</groupId>
<artifactId>jersey-core</artifactId>
<version>${jersey.version}</version>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/13f1a7e0/hadoop-project/src/site/site.xml
----------------------------------------------------------------------
diff --git a/hadoop-project/src/site/site.xml b/hadoop-project/src/site/site.xml
index 232aa78..62e9deb 100644
--- a/hadoop-project/src/site/site.xml
+++ b/hadoop-project/src/site/site.xml
@@ -172,7 +172,6 @@
<item name="DistCp" href="hadoop-distcp/DistCp.html"/>
<item name="GridMix" href="hadoop-gridmix/GridMix.html"/>
<item name="Rumen" href="hadoop-rumen/Rumen.html"/>
- <item name="Resource Estimator Service" href="hadoop-resourceestimator/ResourceEstimator.html"/>
<item name="Scheduler Load Simulator" href="hadoop-sls/SchedulerLoadSimulator.html"/>
<item name="Hadoop Benchmarking" href="hadoop-project-dist/hadoop-common/Benchmarking.html"/>
</menu>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/13f1a7e0/hadoop-tools/hadoop-resourceestimator/README.md
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-resourceestimator/README.md b/hadoop-tools/hadoop-resourceestimator/README.md
deleted file mode 100644
index 1cb8cb5..0000000
--- a/hadoop-tools/hadoop-resourceestimator/README.md
+++ /dev/null
@@ -1,19 +0,0 @@
-Resource Estimator Service
-==========================
-
-Resource Estimator Service can parse the history logs of production jobs, extract their resource consumption skylines in the past runs and predict their resource requirements for the new run.
-
-## Current Status
-
- * Support [Hadoop YARN ResourceManager](http://hadoop.apache.org/docs/current/hadoop-yarn/hadoop-yarn-site/YARN.html) logs.
- * In-memory store for parsed history resource skyline and estimation.
- * A [Linear Programming](https://github.com/optimatika/ojAlgo) based estimator.
- * Provides REST interface to parse logs, query history store and estimations.
-
-## Upcoming features
-
- * UI to query history and edit and save estimations.
- * Persisent store implementation for store (either DB based or distributed key-value like HBase).
- * Integrate directly with the [Hadoop YARN Reservation System](http://hadoop.apache.org/docs/current/hadoop-yarn/hadoop-yarn-site/ReservationSystem.html) to make a recurring reservation based on the estimated resources.
-
-Refer to the [design document](https://issues.apache.org/jira/secure/attachment/12886714/ResourceEstimator-design-v1.pdf) for more details.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/13f1a7e0/hadoop-tools/hadoop-resourceestimator/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-resourceestimator/pom.xml b/hadoop-tools/hadoop-resourceestimator/pom.xml
deleted file mode 100644
index 6790a41..0000000
--- a/hadoop-tools/hadoop-resourceestimator/pom.xml
+++ /dev/null
@@ -1,175 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- ~
- ~ Licensed to the Apache Software Foundation (ASF) under one
- ~ or more contributor license agreements. See the NOTICE file
- ~ distributed with this work for additional information
- ~ regarding copyright ownership. The ASF licenses this file
- ~ to you under the Apache License, Version 2.0 (the
- ~ "License"); you may not use this file except in compliance
- ~ with the License. You may obtain a copy of the License at
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing, software
- ~ distributed under the License is distributed on an "AS IS" BASIS,
- ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- ~ See the License for the specific language governing permissions and
- ~ limitations under the License.
- ~
- -->
-<project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xmlns="http://maven.apache.org/POM/4.0.0"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <modelVersion>4.0.0</modelVersion>
- <parent>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-project</artifactId>
- <version>2.9.0-SNAPSHOT</version>
- <relativePath>../../hadoop-project</relativePath>
- </parent>
- <artifactId>hadoop-resourceestimator</artifactId>
- <version>2.9.0-SNAPSHOT</version>
- <name>Apache Resource Estimator Service</name>
- <packaging>jar</packaging>
- <dependencies>
- <dependency>
- <groupId>com.google.code.gson</groupId>
- <artifactId>gson</artifactId>
- </dependency>
- <dependency>
- <groupId>org.ojalgo</groupId>
- <artifactId>ojalgo</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-common</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-yarn-common</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-yarn-api</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-yarn-server-resourcemanager</artifactId>
- </dependency>
- <dependency>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-api</artifactId>
- </dependency>
- <dependency>
- <groupId>javax.inject</groupId>
- <artifactId>javax.inject</artifactId>
- <version>1</version>
- </dependency>
- <dependency>
- <groupId>com.sun.jersey.jersey-test-framework</groupId>
- <artifactId>jersey-test-framework-grizzly2</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>javax.servlet</groupId>
- <artifactId>servlet-api</artifactId>
- </dependency>
- <dependency>
- <groupId>com.sun.jersey</groupId>
- <artifactId>jersey-server</artifactId>
- </dependency>
- <dependency>
- <groupId>com.sun.jersey</groupId>
- <artifactId>jersey-json</artifactId>
- </dependency>
- <dependency>
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
- <scope>test</scope>
- </dependency>
- </dependencies>
- <build>
- <plugins>
- <plugin>
- <groupId>org.apache.rat</groupId>
- <artifactId>apache-rat-plugin</artifactId>
- <configuration>
- <excludes>
- <exclude>README.md</exclude>
- <exclude>**/*.txt</exclude>
- </excludes>
- </configuration>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-checkstyle-plugin</artifactId>
- <configuration>
- <configLocation>src/config/checkstyle.xml</configLocation>
- </configuration>
- </plugin>
- </plugins>
- </build>
-
- <profiles>
- <profile>
- <id>dist</id>
- <activation>
- <activeByDefault>false</activeByDefault>
- </activation>
- <build>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-assembly-plugin</artifactId>
- <configuration>
- <appendAssemblyId>false</appendAssemblyId>
- <attach>false</attach>
- <finalName>
- ${project.artifactId}-${project.version}
- </finalName>
- <descriptorRefs>
- <descriptorRef>hadoop-resourceestimator
- </descriptorRef>
- </descriptorRefs>
- </configuration>
- <dependencies>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-assemblies</artifactId>
- <version>${project.version}</version>
- </dependency>
- </dependencies>
- <executions>
- <execution>
- <id>dist</id>
- <phase>prepare-package</phase>
- <goals>
- <goal>single</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-dependency-plugin</artifactId>
- <executions>
- <execution>
- <id>deplist</id>
- <phase>compile</phase>
- <goals>
- <goal>list</goal>
- </goals>
- <configuration>
- <outputFile>
- ${project.basedir}/target/hadoop-tools-deps/${project.artifactId}.tools-builtin.txt
- </outputFile>
- </configuration>
- </execution>
- </executions>
- </plugin>
- </plugins>
- </build>
- </profile>
- </profiles>
-</project>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/13f1a7e0/hadoop-tools/hadoop-resourceestimator/src/config/checkstyle.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-resourceestimator/src/config/checkstyle.xml b/hadoop-tools/hadoop-resourceestimator/src/config/checkstyle.xml
deleted file mode 100644
index 7f22659..0000000
--- a/hadoop-tools/hadoop-resourceestimator/src/config/checkstyle.xml
+++ /dev/null
@@ -1,50 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!DOCTYPE module PUBLIC
- "-//Puppy Crawl//DTD Check Configuration 1.2//EN"
- "http://www.puppycrawl.com/dtds/configuration_1_2.dtd">
-
-<!--
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-<!--
- Checkstyle configuration that checks the sun coding conventions from:
-
- - the Java Language Specification at
- http://java.sun.com/docs/books/jls/second_edition/html/index.html
-
- - the Sun Code Conventions at http://java.sun.com/docs/codeconv/
-
- - the Javadoc guidelines at
- http://java.sun.com/j2se/javadoc/writingdoccomments/index.html
-
- - the JDK Api documentation http://java.sun.com/j2se/docs/api/index.html
-
- - some best practices
-
- Checkstyle is very configurable. Be sure to read the documentation at
- http://checkstyle.sf.net (or in your downloaded distribution).
-
- Most Checks are configurable, be sure to consult the documentation.
- To completely disable a check, just comment it out or delete it from the file.
- Finally, it is worth reading the documentation.
--->
-
-<module name="Checker">
- <module name="TreeWalker">
- <module name="AvoidNestedBlocks">
- <property name="id" value="checkstyle:avoidnestedblocks"/>
- <property name="allowInSwitchCase" value="true"/>
- </module>
- </module>
-</module>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/13f1a7e0/hadoop-tools/hadoop-resourceestimator/src/main/bin/estimator.cmd
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-resourceestimator/src/main/bin/estimator.cmd b/hadoop-tools/hadoop-resourceestimator/src/main/bin/estimator.cmd
deleted file mode 100644
index 317025c..0000000
--- a/hadoop-tools/hadoop-resourceestimator/src/main/bin/estimator.cmd
+++ /dev/null
@@ -1,52 +0,0 @@
-@echo off
-@rem Licensed to the Apache Software Foundation (ASF) under one or more
-@rem contributor license agreements. See the NOTICE file distributed with
-@rem this work for additional information regarding copyright ownership.
-@rem The ASF licenses this file to You under the Apache License, Version 2.0
-@rem (the "License"); you may not use this file except in compliance with
-@rem the License. You may obtain a copy of the License at
-@rem
-@rem http://www.apache.org/licenses/LICENSE-2.0
-@rem
-@rem Unless required by applicable law or agreed to in writing, software
-@rem distributed under the License is distributed on an "AS IS" BASIS,
-@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-@rem See the License for the specific language governing permissions and
-@rem limitations under the License.
-@rem
-
-setlocal enabledelayedexpansion
-
-if not defined HADOOP_BIN_PATH (
- set HADOOP_BIN_PATH=%~dp0
-)
-
-if "%HADOOP_BIN_PATH:~-1%" == "\" (
- set HADOOP_BIN_PATH=%HADOOP_BIN_PATH:~0,-1%
-)
-
-set DEFAULT_LIBEXEC_DIR=%HADOOP_BIN_PATH%\..\libexec
-if not defined HADOOP_LIBEXEC_DIR (
- set HADOOP_LIBEXEC_DIR=%DEFAULT_LIBEXEC_DIR%
-)
-
-:main
- @rem CLASSPATH initially contains HADOOP_CONF_DIR
- if not defined HADOOP_CONF_DIR (
- echo No HADOOP_CONF_DIR set.
- echo Please specify it.
- goto :eof
- )
-
- set CLASSPATH=%HADOOP_CONF_DIR%;%CLASSPATH%
-goto :eof
-
-:classpath
- set CLASS=org.apache.hadoop.util.Classpath
- goto :eof
-
-:resourceestimator
- set CLASS=org.apache.hadoop.resourceestimator.service.ResourceEstimatorServer
- goto :eof
-
-endlocal
http://git-wip-us.apache.org/repos/asf/hadoop/blob/13f1a7e0/hadoop-tools/hadoop-resourceestimator/src/main/bin/estimator.sh
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-resourceestimator/src/main/bin/estimator.sh b/hadoop-tools/hadoop-resourceestimator/src/main/bin/estimator.sh
deleted file mode 100644
index 3e1ec27..0000000
--- a/hadoop-tools/hadoop-resourceestimator/src/main/bin/estimator.sh
+++ /dev/null
@@ -1,71 +0,0 @@
-#!/usr/bin/env bash
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License. See accompanying LICENSE file.
-#
-
-## @audience public
-## @stability stable
-function hadoop_usage()
-{
- echo "Usage: estimator.sh"
- #hadoop-daemon.sh. need both start and stop, status (query the status). run as background process.
-}
-
-## @audience public
-## @stability stable
-function calculate_classpath
-{
- hadoop_add_client_opts
- hadoop_add_to_classpath_tools hadoop-resourceestimator
-}
-
-## @audience public
-## @stability stable
-function resourceestimatorcmd_case
-{
- # shellcheck disable=SC2034
- HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
- # shellcheck disable=SC2034
- HADOOP_CLASSNAME='org.apache.hadoop.resourceestimator.service.ResourceEstimatorServer'
-}
-
-# let's locate libexec...
-if [[ -n "${HADOOP_HOME}" ]]; then
- HADOOP_DEFAULT_LIBEXEC_DIR="${HADOOP_HOME}/libexec"
-else
- this="${BASH_SOURCE-$0}"
- bin=$(cd -P -- "$(dirname -- "${this}")" >/dev/null && pwd -P)
- HADOOP_DEFAULT_LIBEXEC_DIR="${bin}/../../../../../libexec"
-fi
-
-HADOOP_LIBEXEC_DIR="${HADOOP_LIBEXEC_DIR:-$HADOOP_DEFAULT_LIBEXEC_DIR}"
-# shellcheck disable=SC2034
-HADOOP_NEW_CONFIG=true
-if [[ -f "${HADOOP_LIBEXEC_DIR}/hadoop-config.sh" ]]; then
- # shellcheck source=./hadoop-common-project/hadoop-common/src/main/bin/hadoop-config.sh
- . "${HADOOP_LIBEXEC_DIR}/hadoop-config.sh"
-else
- echo "ERROR: Cannot execute ${HADOOP_LIBEXEC_DIR}/hadoop-config.sh." 2>&1
- exit 1
-fi
-
-# get arguments
-HADOOP_SUBCMD=$1
-shift
-
-HADOOP_SUBCMD_ARGS=("$@")
-
-resourceestimatorcmd_case "${HADOOP_SUBCMD}" "${HADOOP_SUBCMD_ARGS[@]}"
-
-calculate_classpath
-hadoop_generic_java_subcmd_handler
http://git-wip-us.apache.org/repos/asf/hadoop/blob/13f1a7e0/hadoop-tools/hadoop-resourceestimator/src/main/bin/start-estimator.cmd
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-resourceestimator/src/main/bin/start-estimator.cmd b/hadoop-tools/hadoop-resourceestimator/src/main/bin/start-estimator.cmd
deleted file mode 100644
index 463e91c..0000000
--- a/hadoop-tools/hadoop-resourceestimator/src/main/bin/start-estimator.cmd
+++ /dev/null
@@ -1,37 +0,0 @@
-@echo off
-@rem Licensed to the Apache Software Foundation (ASF) under one or more
-@rem contributor license agreements. See the NOTICE file distributed with
-@rem this work for additional information regarding copyright ownership.
-@rem The ASF licenses this file to You under the Apache License, Version 2.0
-@rem (the "License"); you may not use this file except in compliance with
-@rem the License. You may obtain a copy of the License at
-@rem
-@rem http://www.apache.org/licenses/LICENSE-2.0
-@rem
-@rem Unless required by applicable law or agreed to in writing, software
-@rem distributed under the License is distributed on an "AS IS" BASIS,
-@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-@rem See the License for the specific language governing permissions and
-@rem limitations under the License.
-@rem
-setlocal enabledelayedexpansion
-
-echo starting resource estimator daemons
-
-if not defined HADOOP_BIN_PATH (
- set HADOOP_BIN_PATH=%~dp0
-)
-
-if "%HADOOP_BIN_PATH:~-1%" == "\" (
- set HADOOP_BIN_PATH=%HADOOP_BIN_PATH:~0,-1%
-)
-
-set DEFAULT_LIBEXEC_DIR=%HADOOP_BIN_PATH%\..\libexec
-if not defined HADOOP_LIBEXEC_DIR (
- set HADOOP_LIBEXEC_DIR=%DEFAULT_LIBEXEC_DIR%
-)
-
-@rem start resource estimator
-start "Resource Estimator Service" estimator resourceestimator
-
-endlocal
http://git-wip-us.apache.org/repos/asf/hadoop/blob/13f1a7e0/hadoop-tools/hadoop-resourceestimator/src/main/bin/start-estimator.sh
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-resourceestimator/src/main/bin/start-estimator.sh b/hadoop-tools/hadoop-resourceestimator/src/main/bin/start-estimator.sh
deleted file mode 100644
index bf4328a..0000000
--- a/hadoop-tools/hadoop-resourceestimator/src/main/bin/start-estimator.sh
+++ /dev/null
@@ -1,42 +0,0 @@
-#!/usr/bin/env bash
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License. See accompanying LICENSE file.
-#
-
-# let's locate libexec...
-if [[ -n "${HADOOP_HOME}" ]]; then
- HADOOP_DEFAULT_LIBEXEC_DIR="${HADOOP_HOME}/libexec"
-else
- this="${BASH_SOURCE-$0}"
- bin=$(cd -P -- "$(dirname -- "${this}")" >/dev/null && pwd -P)
- HADOOP_DEFAULT_LIBEXEC_DIR="${bin}/../../../../../libexec"
-fi
-
-HADOOP_LIBEXEC_DIR="${HADOOP_LIBEXEC_DIR:-$HADOOP_DEFAULT_LIBEXEC_DIR}"
-# shellcheck disable=SC2034
-HADOOP_NEW_CONFIG=true
-if [[ -f "${HADOOP_LIBEXEC_DIR}/hadoop-config.sh" ]]; then
- # shellcheck source=./hadoop-common-project/hadoop-common/src/main/bin/hadoop-config.sh
- . "${HADOOP_LIBEXEC_DIR}/hadoop-config.sh"
-else
- echo "ERROR: Cannot execute ${HADOOP_LIBEXEC_DIR}/hadoop-config.sh." 2>&1
- exit 1
-fi
-
-# start resource estimator
-echo "Starting resource estimator"
-hadoop_uservar_su estimator resourceestimator "bin/estimator.sh" \
- --config "${HADOOP_CONF_DIR}" \
- --daemon start \
- resourceestimator
-(( HADOOP_JUMBO_RETCOUNTER=HADOOP_JUMBO_RETCOUNTER + $? ))
http://git-wip-us.apache.org/repos/asf/hadoop/blob/13f1a7e0/hadoop-tools/hadoop-resourceestimator/src/main/bin/stop-estimator.cmd
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-resourceestimator/src/main/bin/stop-estimator.cmd b/hadoop-tools/hadoop-resourceestimator/src/main/bin/stop-estimator.cmd
deleted file mode 100644
index e08c7e0..0000000
--- a/hadoop-tools/hadoop-resourceestimator/src/main/bin/stop-estimator.cmd
+++ /dev/null
@@ -1,37 +0,0 @@
-@echo off
-@rem Licensed to the Apache Software Foundation (ASF) under one or more
-@rem contributor license agreements. See the NOTICE file distributed with
-@rem this work for additional information regarding copyright ownership.
-@rem The ASF licenses this file to You under the Apache License, Version 2.0
-@rem (the "License"); you may not use this file except in compliance with
-@rem the License. You may obtain a copy of the License at
-@rem
-@rem http://www.apache.org/licenses/LICENSE-2.0
-@rem
-@rem Unless required by applicable law or agreed to in writing, software
-@rem distributed under the License is distributed on an "AS IS" BASIS,
-@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-@rem See the License for the specific language governing permissions and
-@rem limitations under the License.
-@rem
-setlocal enabledelayedexpansion
-
-echo stopping resource estimator daemons
-
-if not defined HADOOP_BIN_PATH (
- set HADOOP_BIN_PATH=%~dp0
-)
-
-if "%HADOOP_BIN_PATH:~-1%" == "\" (
- set HADOOP_BIN_PATH=%HADOOP_BIN_PATH:~0,-1%
-)
-
-set DEFAULT_LIBEXEC_DIR=%HADOOP_BIN_PATH%\..\libexec
-if not defined HADOOP_LIBEXEC_DIR (
- set HADOOP_LIBEXEC_DIR=%DEFAULT_LIBEXEC_DIR%
-)
-
-@rem start resource estimator
-stop "Resource Estimator Service" estimator resourceestimator
-
-endlocal
http://git-wip-us.apache.org/repos/asf/hadoop/blob/13f1a7e0/hadoop-tools/hadoop-resourceestimator/src/main/bin/stop-estimator.sh
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-resourceestimator/src/main/bin/stop-estimator.sh b/hadoop-tools/hadoop-resourceestimator/src/main/bin/stop-estimator.sh
deleted file mode 100644
index e78c798..0000000
--- a/hadoop-tools/hadoop-resourceestimator/src/main/bin/stop-estimator.sh
+++ /dev/null
@@ -1,42 +0,0 @@
-#!/usr/bin/env bash
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License. See accompanying LICENSE file.
-#
-
-# let's locate libexec...
-if [[ -n "${HADOOP_HOME}" ]]; then
- HADOOP_DEFAULT_LIBEXEC_DIR="${HADOOP_HOME}/libexec"
-else
- this="${BASH_SOURCE-$0}"
- bin=$(cd -P -- "$(dirname -- "${this}")" >/dev/null && pwd -P)
- HADOOP_DEFAULT_LIBEXEC_DIR="${bin}/../../../../../libexec"
-fi
-
-HADOOP_LIBEXEC_DIR="${HADOOP_LIBEXEC_DIR:-$HADOOP_DEFAULT_LIBEXEC_DIR}"
-# shellcheck disable=SC2034
-HADOOP_NEW_CONFIG=true
-if [[ -f "${HADOOP_LIBEXEC_DIR}/hadoop-config.sh" ]]; then
- # shellcheck source=./hadoop-common-project/hadoop-common/src/main/bin/hadoop-config.sh
- . "${HADOOP_LIBEXEC_DIR}/hadoop-config.sh"
-else
- echo "ERROR: Cannot execute ${HADOOP_LIBEXEC_DIR}/hadoop-config.sh." 2>&1
- exit 1
-fi
-
-# stop resource estimator
-echo "Stopping resource estimator"
-hadoop_uservar_su estimator resourceestimator "bin/estimator.sh" \
- --config "${HADOOP_CONF_DIR}" \
- --daemon stop \
- resourceestimator
-(( HADOOP_JUMBO_RETCOUNTER=HADOOP_JUMBO_RETCOUNTER + $? ))
http://git-wip-us.apache.org/repos/asf/hadoop/blob/13f1a7e0/hadoop-tools/hadoop-resourceestimator/src/main/conf/resourceestimator-config.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-resourceestimator/src/main/conf/resourceestimator-config.xml b/hadoop-tools/hadoop-resourceestimator/src/main/conf/resourceestimator-config.xml
deleted file mode 100644
index f78085a..0000000
--- a/hadoop-tools/hadoop-resourceestimator/src/main/conf/resourceestimator-config.xml
+++ /dev/null
@@ -1,85 +0,0 @@
-<?xml version="1.0"?>
-<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
- http://www.apache.org/licenses/LICENSE-2.0
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<configuration>
- <property>
- <name>resourceestimator.solver.lp.alpha</name>
- <value>0.1</value>
- <description>
- The resource estimator has an integrated Linear Programming solver to make the prediction, and this parameter tunes the tradeoff between resource over-allocation and under-allocation in the Linear Programming model. This parameter varies from 0 to 1, and a larger alpha value means the model minimizes over-allocation better. Default value is 0.1.
- </description>
- </property>
- <property>
- <name>resourceestimator.solver.lp.beta</name>
- <value>0.1</value>
- <description>
- This parameter controls the generalization of the Linear Programming model. This parameter varies from 0 to 1. Deafult value is 0.1.
- </description>
- </property>
- <property>
- <name>resourceestimator.solver.lp.minJobRuns</name>
- <value>2</value>
- <description>
- The minimum number of job runs required in order to make the prediction. Default value is 2.
- </description>
- </property>
- <property>
- <name>resourceestimator.timeInterval</name>
- <value>5</value>
- <description>
- The time length which is used to discretize job execution into intervals. Note that the estimator makes resource allocation prediction for each interval. A smaller time interval has more fine-grained granularity for prediction, but it also takes longer time and more space for prediction. Default value is 5 (seconds).
- </description>
- </property>
- <property>
- <name>resourceestimator.translator.line-parser</name>
- <value>org.apache.hadoop.resourceestimator.translator.impl.NativeSingleLineParser</value>
- <description>
- The class name of the translator single-line parser, which parses a single line in the log. Default value is org.apache.hadoop.resourceestimator.translator.impl.NativeSingleLineParser, which can parse one line in the sample log. Note that if users want to parse Hadoop Resource Manager logs, they need to set the value to be org.apache.hadoop.resourceestimator.translator.impl.RmSingleLineParser. If they want to implement single-line parser to parse their customized log file, they need to change this value accordingly.
- </description>
- </property>
- <!-- Optional configuration parameters -->
- <!--
- <property>
- <name>resourceestimator.solver.provider</name>
- <value>org.apache.hadoop.resourceestimator.solver.impl.LpSolver</value>
- <description>
- The class name of the solver provider. Default value is org.apache.hadoop.resourceestimator.solver.impl.LpSolver, which incorporates a Linear Programming model to make the prediction. If users want to implement their own models, they need to change this value accordingly.
- </description>
- </property>
- <property>
- <name>resourceestimator.skylinestore.provider</name>
- <value>org.apache.hadoop.resourceestimator.skylinestore.impl.InMemoryStore</value>
- <description>
- The class name of the skylinestore provider. Default value is org.apache.hadoop.resourceestimator.skylinestore.impl.InMemoryStore, which is an in-memory implementation of skylinestore. If users want to use their own skylinestore implementation, they need to change this value accordingly.
- </description>
- </property>
- <property>
- <name>resourceestimator.translator.provider</name>
- <value>org.apache.hadoop.resourceestimator.translator.impl.BaseLogParser</value>
- <description>
- The class name of the translator provider. Default value is org.apache.hadoop.resourceestimator.translator.impl.BaseLogParser, which extracts resourceskylines from log streams. If users want to use their own translator implementation, they need to change this value accordingly.
- </description>
- </property>
- <property>
- <name>resourceestimator.service-port</name>
- <value>9998</value>
- <description>
- The port which ResourceEstimatorService listens to. The default value is 9998.
- </description>
- </property>
- -->
-</configuration>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/13f1a7e0/hadoop-tools/hadoop-resourceestimator/src/main/data/resourceEstimatorService.txt
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-resourceestimator/src/main/data/resourceEstimatorService.txt b/hadoop-tools/hadoop-resourceestimator/src/main/data/resourceEstimatorService.txt
deleted file mode 100644
index 15fb84e..0000000
--- a/hadoop-tools/hadoop-resourceestimator/src/main/data/resourceEstimatorService.txt
+++ /dev/null
@@ -1,2 +0,0 @@
-tpch_q12 50 1462148517000 1462148527345 1462148952345 tpch_q12_0 8192:0| 1:0|1:5|1074:10|2538:15|2468:20|
-tpch_q12 50 1462148949000 1462148960455 1462149205455 tpch_q12_1 8192:0| 1:0|1:5|794:10|2517:15|2484:20|
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/13f1a7e0/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/common/api/RecurrenceId.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/common/api/RecurrenceId.java b/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/common/api/RecurrenceId.java
deleted file mode 100644
index 01ef977..0000000
--- a/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/common/api/RecurrenceId.java
+++ /dev/null
@@ -1,95 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package org.apache.hadoop.resourceestimator.common.api;
-
-/**
- * RecurrenceId is the id for the recurring pipeline jobs.
- * <p> We assume that the pipeline job can be uniquely identified with
- * {pipelineId, runId}.
- */
-public class RecurrenceId {
- /**
- * pipelineId is the unique id for the pipeline jobs.
- */
- private String pipelineId;
- /**
- * runId is the unique instance id for the pipeline job in one run, and it
- * will change across runs.
- */
- private String runId;
- // TODO: we may addHistory more ids of the pipeline jobs to identify them.
-
- /**
- * Constructor.
- *
- * @param pipelineIdConfig the unique id for the pipeline jobs.
- * @param runIdConfig the unique instance id for the pipeline job in one run.
- */
- public RecurrenceId(final String pipelineIdConfig, final String runIdConfig) {
- this.pipelineId = pipelineIdConfig;
- this.runId = runIdConfig;
- }
-
- /**
- * Return the pipelineId for the pipeline jobs.
- *
- * @return the pipelineId.
- */
- public final String getPipelineId() {
- return pipelineId;
- }
-
- public void setPipelineId(String pipelineId) {
- this.pipelineId = pipelineId;
- }
-
- /**
- * Return the runId for the pipeline job in one run.
- *
- * @return the runId.
- */
- public final String getRunId() {
- return runId;
- }
-
- public void setRunId(String runId) {
- this.runId = runId;
- }
-
- @Override public final String toString() {
- return String.format("{pipelineId: %s, runId: %s}", pipelineId, runId);
- }
-
- @Override public final int hashCode() {
- return getPipelineId().hashCode() ^ getRunId().hashCode();
- }
-
- @Override public final boolean equals(final Object obj) {
- if (this == obj) {
- return true;
- }
- if ((obj == null) || (getClass() != obj.getClass())) {
- return false;
- }
- final RecurrenceId other = (RecurrenceId) obj;
- return pipelineId.equals(other.pipelineId) && runId.equals(other.runId);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/13f1a7e0/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/common/api/ResourceSkyline.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/common/api/ResourceSkyline.java b/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/common/api/ResourceSkyline.java
deleted file mode 100644
index a4bc69f..0000000
--- a/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/common/api/ResourceSkyline.java
+++ /dev/null
@@ -1,211 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package org.apache.hadoop.resourceestimator.common.api;
-
-import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.server.resourcemanager.reservation.RLESparseResourceAllocation;
-
-/**
- * ResourceSkyline records the job identification information as well as job's
- * requested {@code
- * container}s information during its lifespan.
- */
-public class ResourceSkyline {
- /**
- * The auto-generated {@code ApplicationId} in job's one run.
- * <p>
- * <p>
- * For a pipeline job, we assume the {@code jobId} changes each time we run
- * the pipeline job.
- */
- private String jobId;
- /**
- * The input data size of the job.
- */
- private double jobInputDataSize;
- /**
- * Job submission time. Different logs could have different time format, so we
- * store the original string directly extracted from logs.
- */
- private long jobSubmissionTime;
- /**
- * Job finish time. Different logs could have different time format, so we
- * store the original string directly extracted from logs.
- */
- private long jobFinishTime;
- /**
- * The resource spec of containers allocated to the job.
- */
- private Resource containerSpec;
- /**
- * The list of {@link Resource} allocated to the job in its lifespan.
- */
- private RLESparseResourceAllocation skylineList;
- // TODO
- // We plan to record pipeline job's actual resource consumptions in one run
- // here.
- // TODO
- // We might need to addHistory more features to the ResourceSkyline, such as
- // users, VC, etc.
-
- /**
- * Constructor.
- *
- * @param jobIdConfig the id of the job.
- * @param jobInputDataSizeConfig the input data size of the job.
- * @param jobSubmissionTimeConfig the submission time of the job.
- * @param jobFinishTimeConfig the finish time of the job.
- * @param containerSpecConfig the resource spec of containers allocated
- * to the job.
- * @param skylineListConfig the list of {@link Resource} allocated in
- * one run.
- */
- public ResourceSkyline(final String jobIdConfig,
- final double jobInputDataSizeConfig, final long jobSubmissionTimeConfig,
- final long jobFinishTimeConfig, final Resource containerSpecConfig,
- final RLESparseResourceAllocation skylineListConfig) {
- this.jobId = jobIdConfig;
- this.jobInputDataSize = jobInputDataSizeConfig;
- this.jobSubmissionTime = jobSubmissionTimeConfig;
- this.jobFinishTime = jobFinishTimeConfig;
- this.containerSpec = containerSpecConfig;
- this.skylineList = skylineListConfig;
- }
-
- /**
- * Empty constructor.
- */
- public ResourceSkyline() {
- }
-
- /**
- * Get the id of the job.
- *
- * @return the id of this job.
- */
- public final String getJobId() {
- return jobId;
- }
-
- /**
- * Set jobId.
- *
- * @param jobIdConfig jobId.
- */
- public final void setJobId(final String jobIdConfig) {
- this.jobId = jobIdConfig;
- }
-
- /**
- * Get the job's input data size.
- *
- * @return job's input data size.
- */
- public final double getJobInputDataSize() {
- return jobInputDataSize;
- }
-
- /**
- * Set jobInputDataSize.
- *
- * @param jobInputDataSizeConfig jobInputDataSize.
- */
- public final void setJobInputDataSize(final double jobInputDataSizeConfig) {
- this.jobInputDataSize = jobInputDataSizeConfig;
- }
-
- /**
- * Get the job's submission time.
- *
- * @return job's submission time.
- */
- public final long getJobSubmissionTime() {
- return jobSubmissionTime;
- }
-
- /**
- * Set jobSubmissionTime.
- *
- * @param jobSubmissionTimeConfig jobSubmissionTime.
- */
- public final void setJobSubmissionTime(final long jobSubmissionTimeConfig) {
- this.jobSubmissionTime = jobSubmissionTimeConfig;
- }
-
- /**
- * Get the job's finish time.
- *
- * @return job's finish time.
- */
- public final long getJobFinishTime() {
- return jobFinishTime;
- }
-
- /**
- * Set jobFinishTime.
- *
- * @param jobFinishTimeConfig jobFinishTime.
- */
- public final void setJobFinishTime(final long jobFinishTimeConfig) {
- this.jobFinishTime = jobFinishTimeConfig;
- }
-
- /**
- * Get the resource spec of the job's allocated {@code container}s.
- * <p> Key assumption: during job's lifespan, its allocated {@code container}s
- * have the same {@link Resource} spec.
- *
- * @return the {@link Resource} spec of the job's allocated
- * {@code container}s.
- */
- public final Resource getContainerSpec() {
- return containerSpec;
- }
-
- /**
- * Set containerSpec.
- *
- * @param containerSpecConfig containerSpec.
- */
- public final void setContainerSpec(final Resource containerSpecConfig) {
- this.containerSpec = containerSpecConfig;
- }
-
- /**
- * Get the list of {@link Resource}s allocated to the job.
- *
- * @return the {@link RLESparseResourceAllocation} which contains the list of
- * {@link Resource}s allocated to the job.
- */
- public final RLESparseResourceAllocation getSkylineList() {
- return skylineList;
- }
-
- /**
- * Set skylineList.
- *
- * @param skylineListConfig skylineList.
- */
- public final void setSkylineList(
- final RLESparseResourceAllocation skylineListConfig) {
- this.skylineList = skylineListConfig;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/13f1a7e0/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/common/api/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/common/api/package-info.java b/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/common/api/package-info.java
deleted file mode 100644
index a7c7527..0000000
--- a/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/common/api/package-info.java
+++ /dev/null
@@ -1,23 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * The common APIs for the resource estimator.
- */
-
-package org.apache.hadoop.resourceestimator.common.api;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/13f1a7e0/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/common/config/ResourceEstimatorConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/common/config/ResourceEstimatorConfiguration.java b/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/common/config/ResourceEstimatorConfiguration.java
deleted file mode 100644
index bb041b6..0000000
--- a/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/common/config/ResourceEstimatorConfiguration.java
+++ /dev/null
@@ -1,125 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package org.apache.hadoop.resourceestimator.common.config;
-
-import org.apache.hadoop.resourceestimator.skylinestore.impl.InMemoryStore;
-import org.apache.hadoop.resourceestimator.solver.impl.LpSolver;
-import org.apache.hadoop.resourceestimator.translator.impl.BaseLogParser;
-import org.apache.hadoop.resourceestimator.translator.impl.NativeSingleLineParser;
-
-/**
- * Defines configuration keys for ResourceEstimatorServer.
- */
-public final class ResourceEstimatorConfiguration {
-
- /**
- * The location of the configuration file for ResourceEstimatorService.
- */
- public static final String CONFIG_FILE = "resourceestimator-config.xml";
-
- /**
- * The URI for ResourceEstimatorService.
- */
- public static final String SERVICE_URI = "http://0.0.0.0/";
-
- /**
- * The port which ResourceEstimatorService listens to.
- */
- public static final String SERVICE_PORT = "resourceestimator.service-port";
-
- /**
- * Default port number of ResourceEstimatorService.
- */
- public static final int DEFAULT_SERVICE_PORT = 9998;
-
- /**
- * The class name of the skylinestore provider.
- */
- public static final String SKYLINESTORE_PROVIDER =
- "resourceestimator.skylinestore.provider";
-
- /**
- * Default value for skylinestore provider, which is an in-memory implementation of skylinestore.
- */
- public static final String DEFAULT_SKYLINESTORE_PROVIDER =
- InMemoryStore.class.getName();
-
- /**
- * The class name of the translator provider.
- */
- public static final String TRANSLATOR_PROVIDER =
- "resourceestimator.translator.provider";
-
- /**
- * Default value for translator provider, which extracts resourceskylines from log streams.
- */
- public static final String DEFAULT_TRANSLATOR_PROVIDER =
- BaseLogParser.class.getName();
-
- /**
- * The class name of the translator single-line parser, which parses a single line in the log.
- */
- public static final String TRANSLATOR_LINE_PARSER =
- "resourceestimator.translator.line-parser";
-
- /**
- * Default value for translator single-line parser, which can parse one line in the sample log.
- */
- public static final String DEFAULT_TRANSLATOR_LINE_PARSER =
- NativeSingleLineParser.class.getName();
-
- /**
- * The class name of the solver provider.
- */
- public static final String SOLVER_PROVIDER =
- "resourceestimator.solver.provider";
-
- /**
- * Default value for solver provider, which incorporates a Linear Programming model to make the prediction.
- */
- public static final String DEFAULT_SOLVER_PROVIDER = LpSolver.class.getName();
-
- /**
- * The time length which is used to discretize job execution into intervals.
- */
- public static final String TIME_INTERVAL_KEY =
- "resourceestimator.timeInterval";
-
- /**
- * The parameter which tunes the tradeoff between resource over-allocation and under-allocation in the Linear Programming model.
- */
- public static final String SOLVER_ALPHA_KEY =
- "resourceestimator.solver.lp.alpha";
-
- /**
- * This parameter which controls the generalization of the Linear Programming model.
- */
- public static final String SOLVER_BETA_KEY =
- "resourceestimator.solver.lp.beta";
-
- /**
- * The minimum number of job runs required in order to make the prediction.
- */
- public static final String SOLVER_MIN_JOB_RUN_KEY =
- "resourceestimator.solver.lp.minJobRuns";
-
- private ResourceEstimatorConfiguration() {}
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/13f1a7e0/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/common/config/ResourceEstimatorUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/common/config/ResourceEstimatorUtil.java b/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/common/config/ResourceEstimatorUtil.java
deleted file mode 100644
index c00dd00..0000000
--- a/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/common/config/ResourceEstimatorUtil.java
+++ /dev/null
@@ -1,81 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package org.apache.hadoop.resourceestimator.common.config;
-
-import java.lang.reflect.Constructor;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.resourceestimator.common.exception.ResourceEstimatorException;
-
-/**
- * General resourceestimator utils.
- */
-public final class ResourceEstimatorUtil {
-
- private static final Class<?>[] EMPTY_ARRAY = new Class[0];
-
- private ResourceEstimatorUtil() {}
-
- /**
- * Helper method to create instances of Object using the class name specified
- * in the configuration object.
- *
- * @param conf the yarn configuration
- * @param configuredClassName the configuration provider key
- * @param defaultValue the default implementation class
- * @param type the required interface/base class
- * @param <T> The type of the instance to create
- * @return the instances created
- * @throws ResourceEstimatorException if the provider initialization fails.
- */
- @SuppressWarnings("unchecked") public static <T> T createProviderInstance(
- Configuration conf, String configuredClassName, String defaultValue,
- Class<T> type) throws ResourceEstimatorException {
- String className = conf.get(configuredClassName);
- if (className == null) {
- className = defaultValue;
- }
- try {
- Class<?> concreteClass = Class.forName(className);
- if (type.isAssignableFrom(concreteClass)) {
- Constructor<T> meth =
- (Constructor<T>) concreteClass.getDeclaredConstructor(EMPTY_ARRAY);
- meth.setAccessible(true);
- return meth.newInstance();
- } else {
- StringBuilder errMsg = new StringBuilder();
- errMsg.append("Class: ").append(className).append(" not instance of ")
- .append(type.getCanonicalName());
- throw new ResourceEstimatorException(errMsg.toString());
- }
- } catch (ClassNotFoundException e) {
- StringBuilder errMsg = new StringBuilder();
- errMsg.append("Could not instantiate : ").append(className)
- .append(" due to exception: ").append(e.getCause());
- throw new ResourceEstimatorException(errMsg.toString());
- } catch (ReflectiveOperationException e) {
- StringBuilder errMsg = new StringBuilder();
- errMsg.append("Could not instantiate : ").append(className)
- .append(" due to exception: ").append(e.getCause());
- throw new ResourceEstimatorException(errMsg.toString());
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/13f1a7e0/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/common/config/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/common/config/package-info.java b/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/common/config/package-info.java
deleted file mode 100644
index ecf140a..0000000
--- a/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/common/config/package-info.java
+++ /dev/null
@@ -1,23 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * The common configuration for the resource estimator.
- */
-
-package org.apache.hadoop.resourceestimator.common.config;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/13f1a7e0/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/common/exception/ResourceEstimatorException.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/common/exception/ResourceEstimatorException.java b/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/common/exception/ResourceEstimatorException.java
deleted file mode 100644
index ba5666d..0000000
--- a/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/common/exception/ResourceEstimatorException.java
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package org.apache.hadoop.resourceestimator.common.exception;
-
-/**
- * Exception thrown by ResourceEstimatorServer utility classes.
- */
-public class ResourceEstimatorException extends Exception {
-
- public ResourceEstimatorException(String message) {
- super(message);
- }
-
- public ResourceEstimatorException(String message, Exception ex) {
- super(message, ex);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/13f1a7e0/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/common/exception/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/common/exception/package-info.java b/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/common/exception/package-info.java
deleted file mode 100644
index 6ebecb6..0000000
--- a/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/common/exception/package-info.java
+++ /dev/null
@@ -1,23 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * The common exception for the resource estimator.
- */
-
-package org.apache.hadoop.resourceestimator.common.exception;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/13f1a7e0/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/common/serialization/RLESparseResourceAllocationSerDe.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/common/serialization/RLESparseResourceAllocationSerDe.java b/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/common/serialization/RLESparseResourceAllocationSerDe.java
deleted file mode 100644
index 379dc3a..0000000
--- a/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/common/serialization/RLESparseResourceAllocationSerDe.java
+++ /dev/null
@@ -1,77 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package org.apache.hadoop.resourceestimator.common.serialization;
-
-import java.lang.reflect.Type;
-import java.util.NavigableMap;
-
-import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.server.resourcemanager.reservation.RLESparseResourceAllocation;
-import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator;
-import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
-
-import com.google.gson.Gson;
-import com.google.gson.GsonBuilder;
-import com.google.gson.JsonDeserializationContext;
-import com.google.gson.JsonDeserializer;
-import com.google.gson.JsonElement;
-import com.google.gson.JsonObject;
-import com.google.gson.JsonParseException;
-import com.google.gson.JsonSerializationContext;
-import com.google.gson.JsonSerializer;
-import com.google.gson.reflect.TypeToken;
-
-/**
- * Serialize/deserialize RLESparseResourceAllocation object to/from JSON.
- */
-public class RLESparseResourceAllocationSerDe
- implements JsonSerializer<RLESparseResourceAllocation>,
- JsonDeserializer<RLESparseResourceAllocation> {
- private static final String KEY = "resourceAllocation";
- private final Gson gson =
- new GsonBuilder().registerTypeAdapter(Resource.class, new ResourceSerDe())
- .create();
- private final Type type = new TypeToken<NavigableMap<Long, Resource>>() {
- }.getType();
- private final ResourceCalculator resourceCalculator =
- new DefaultResourceCalculator();
-
- @Override public final JsonElement serialize(
- final RLESparseResourceAllocation resourceAllocation,
- final Type typeOfSrc, final JsonSerializationContext context) {
- NavigableMap<Long, Resource> myMap = resourceAllocation.getCumulative();
- JsonObject jo = new JsonObject();
- JsonElement element = gson.toJsonTree(myMap, type);
- jo.add(KEY, element);
-
- return jo;
- }
-
- @Override public final RLESparseResourceAllocation deserialize(
- final JsonElement json, final Type typeOfT,
- final JsonDeserializationContext context) throws JsonParseException {
- NavigableMap<Long, Resource> resAllocation =
- gson.fromJson(json.getAsJsonObject().get(KEY), type);
- RLESparseResourceAllocation rleSparseResourceAllocation =
- new RLESparseResourceAllocation(resAllocation, resourceCalculator);
- return rleSparseResourceAllocation;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/13f1a7e0/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/common/serialization/ResourceSerDe.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/common/serialization/ResourceSerDe.java b/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/common/serialization/ResourceSerDe.java
deleted file mode 100644
index 22b079c..0000000
--- a/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/common/serialization/ResourceSerDe.java
+++ /dev/null
@@ -1,61 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package org.apache.hadoop.resourceestimator.common.serialization;
-
-import java.lang.reflect.Type;
-
-import org.apache.hadoop.yarn.api.records.Resource;
-
-import com.google.gson.JsonDeserializationContext;
-import com.google.gson.JsonDeserializer;
-import com.google.gson.JsonElement;
-import com.google.gson.JsonObject;
-import com.google.gson.JsonParseException;
-import com.google.gson.JsonSerializationContext;
-import com.google.gson.JsonSerializer;
-
-/**
- * Serialize/deserialize Resource object to/from JSON.
- */
-public class ResourceSerDe
- implements JsonSerializer<Resource>, JsonDeserializer<Resource> {
- private static final String KEY1 = "memory";
- private static final String KEY2 = "vcores";
-
- @Override public final JsonElement serialize(final Resource resource,
- final Type type, final JsonSerializationContext context) {
- JsonObject jo = new JsonObject();
- jo.addProperty(KEY1, resource.getMemorySize());
- jo.addProperty(KEY2, resource.getVirtualCores());
- return jo;
- }
-
- @Override public final Resource deserialize(final JsonElement json,
- final Type type, final JsonDeserializationContext context)
- throws JsonParseException {
- JsonObject jo = json.getAsJsonObject();
- long mem = jo.getAsJsonPrimitive(KEY1).getAsLong();
- int vcore = jo.getAsJsonPrimitive(KEY2).getAsInt();
- Resource resource = Resource.newInstance(mem, vcore);
-
- return resource;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/13f1a7e0/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/common/serialization/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/common/serialization/package-info.java b/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/common/serialization/package-info.java
deleted file mode 100644
index 06957b3..0000000
--- a/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/common/serialization/package-info.java
+++ /dev/null
@@ -1,24 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-/**
- * SkylineStore serialization module.
- */
-package org.apache.hadoop.resourceestimator.common.serialization;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/13f1a7e0/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/service/ResourceEstimatorServer.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/service/ResourceEstimatorServer.java b/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/service/ResourceEstimatorServer.java
deleted file mode 100644
index 27a5864..0000000
--- a/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/service/ResourceEstimatorServer.java
+++ /dev/null
@@ -1,146 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package org.apache.hadoop.resourceestimator.service;
-
-import java.io.IOException;
-import java.net.URI;
-
-import javax.ws.rs.core.UriBuilder;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.http.HttpServer2;
-import org.apache.hadoop.resourceestimator.common.config.ResourceEstimatorConfiguration;
-import org.apache.hadoop.service.CompositeService;
-import org.apache.hadoop.util.ShutdownHookManager;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
-import org.apache.hadoop.yarn.webapp.YarnJacksonJaxbJsonProvider;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * A simple embedded Hadoop HTTP server.
- */
-public final class ResourceEstimatorServer extends CompositeService {
- private static final Logger LOGGER =
- LoggerFactory.getLogger(ResourceEstimatorServer.class.getName());
- private HttpServer2 webServer;
- private static URI baseURI;
-
- public ResourceEstimatorServer() {
- super(ResourceEstimatorServer.class.getName());
- }
-
- private static URI getBaseURI(Configuration config) {
- baseURI = UriBuilder.fromUri(ResourceEstimatorConfiguration.SERVICE_URI)
- .port(getPort(config)).build();
- return baseURI;
- }
-
- private static int getPort(Configuration config) {
- return config.getInt(ResourceEstimatorConfiguration.SERVICE_PORT,
- ResourceEstimatorConfiguration.DEFAULT_SERVICE_PORT);
- }
-
- @Override protected void serviceInit(Configuration conf) throws Exception {
- super.serviceInit(conf);
- }
-
- @Override protected void serviceStart() throws Exception {
- super.serviceStart();
- startResourceEstimatorApp();
- }
-
- private void join() {
- // keep the main thread that started the server up until it receives a stop
- // signal
- if (webServer != null) {
- try {
- webServer.join();
- } catch (InterruptedException ignore) {
- }
- }
- }
-
- @Override protected void serviceStop() throws Exception {
- if (webServer != null) {
- webServer.stop();
- }
- super.serviceStop();
- }
-
- private void startResourceEstimatorApp() throws IOException {
- Configuration config = new YarnConfiguration();
- config.addResource(ResourceEstimatorConfiguration.CONFIG_FILE);
- HttpServer2.Builder builder =
- new HttpServer2.Builder().setName("ResourceEstimatorServer")
- .setConf(config)
- //.setFindPort(true)
- .addEndpoint(getBaseURI(config));
- webServer = builder.build();
- webServer.addJerseyResourcePackage(
- ResourceEstimatorService.class.getPackage().getName() + ";"
- + GenericExceptionHandler.class.getPackage().getName() + ";"
- + YarnJacksonJaxbJsonProvider.class.getPackage().getName(), "/*");
- webServer.start();
- }
-
- /**
- * Start embedded Hadoop HTTP server.
- *
- * @return an instance of the started HTTP server.
- * @throws IOException in case there is an error while starting server.
- */
- static ResourceEstimatorServer startResourceEstimatorServer()
- throws IOException, InterruptedException {
- Configuration config = new YarnConfiguration();
- config.addResource(ResourceEstimatorConfiguration.CONFIG_FILE);
- ResourceEstimatorServer resourceEstimatorServer = null;
- try {
- resourceEstimatorServer = new ResourceEstimatorServer();
- ShutdownHookManager.get().addShutdownHook(
- new CompositeServiceShutdownHook(resourceEstimatorServer), 30);
- resourceEstimatorServer.init(config);
- resourceEstimatorServer.start();
- } catch (Throwable t) {
- LOGGER.error("Error starting ResourceEstimatorServer", t);
- }
-
- return resourceEstimatorServer;
- }
-
- public static void main(String[] args)
- throws InterruptedException, IOException {
- ResourceEstimatorServer server = startResourceEstimatorServer();
- server.join();
- }
-
- /**
- * Stop embedded Hadoop HTTP server.
- *
- * @throws Exception in case the HTTP server fails to shut down.
- */
- public void shutdown() throws Exception {
- LOGGER.info("Stopping resourceestimator service at: {}.",
- baseURI.toString());
- webServer.stop();
- }
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org