You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by se...@apache.org on 2015/05/29 13:46:44 UTC
[09/38] incubator-ignite git commit: #IGNITE-857 Added web-server.
Support set ignite version.
#IGNITE-857 Added web-server. Support set ignite version.
Project: http://git-wip-us.apache.org/repos/asf/incubator-ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-ignite/commit/4b482187
Tree: http://git-wip-us.apache.org/repos/asf/incubator-ignite/tree/4b482187
Diff: http://git-wip-us.apache.org/repos/asf/incubator-ignite/diff/4b482187
Branch: refs/heads/ignite-943
Commit: 4b482187871184e4e8d8086aae870ff47c057275
Parents: ae8bcf8
Author: nikolay tikhonov <nt...@gridgain.com>
Authored: Mon May 25 17:10:50 2015 +0300
Committer: nikolay tikhonov <nt...@gridgain.com>
Committed: Mon May 25 17:10:50 2015 +0300
----------------------------------------------------------------------
modules/mesos/pom.xml | 22 ++
.../apache/ignite/mesos/ClusterProperties.java | 263 +++++++++++++++++++
.../apache/ignite/mesos/ClusterResources.java | 160 -----------
.../apache/ignite/mesos/IgniteFramework.java | 64 +++--
.../apache/ignite/mesos/IgniteScheduler.java | 67 +++--
.../ignite/mesos/resource/IgniteProvider.java | 234 +++++++++++++++++
.../mesos/resource/ResourceController.java | 130 +++++++++
.../ignite/mesos/resource/ResourceProvider.java | 120 +++++++++
.../main/resources/ignite-default-config.xml | 35 +++
modules/mesos/src/main/resources/log4j2.xml | 35 +++
10 files changed, 924 insertions(+), 206 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/4b482187/modules/mesos/pom.xml
----------------------------------------------------------------------
diff --git a/modules/mesos/pom.xml b/modules/mesos/pom.xml
index 5ce3e5c..4aa0dae 100644
--- a/modules/mesos/pom.xml
+++ b/modules/mesos/pom.xml
@@ -27,6 +27,10 @@
<artifactId>ignite-mesos</artifactId>
<version>1.1.0-SNAPSHOT</version>
+ <properties>
+ <version.grizzly>2.16</version.grizzly>
+ </properties>
+
<dependencies>
<dependency>
<groupId>org.apache.mesos</groupId>
@@ -41,6 +45,24 @@
</dependency>
<dependency>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-core</artifactId>
+ <version>2.0.2</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-slf4j-impl</artifactId>
+ <version>2.0.2</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.glassfish.jersey.containers</groupId>
+ <artifactId>jersey-container-grizzly2-http</artifactId>
+ <version>${version.grizzly}</version>
+ </dependency>
+
+ <dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>4.11</version>
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/4b482187/modules/mesos/src/main/java/org/apache/ignite/mesos/ClusterProperties.java
----------------------------------------------------------------------
diff --git a/modules/mesos/src/main/java/org/apache/ignite/mesos/ClusterProperties.java b/modules/mesos/src/main/java/org/apache/ignite/mesos/ClusterProperties.java
new file mode 100644
index 0000000..bb7f7a4
--- /dev/null
+++ b/modules/mesos/src/main/java/org/apache/ignite/mesos/ClusterProperties.java
@@ -0,0 +1,263 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.mesos;
+
+import java.io.*;
+import java.util.*;
+
+/**
+ * Cluster settings.
+ */
+public class ClusterProperties {
+ /** Unlimited. */
+ public static final double UNLIMITED = -1;
+
+ /** */
+ public static final String MESOS_MASTER_URL = "MESOS_MASTER_URL";
+
+ /** */
+ public static final String DEFAULT_MESOS_MASTER_URL = "zk://localhost:2181/mesos";
+
+ /** Mesos master url. */
+ private String mesosUrl = DEFAULT_MESOS_MASTER_URL;
+
+ /** */
+ public static final String IGNITE_RESOURCE_CPU_CORES = "IGNITE_RESOURCE_CPU_CORES";
+
+ /** CPU limit. */
+ private double cpu = UNLIMITED;
+
+ /** */
+ public static final String IGNITE_RESOURCE_MEM_MB = "IGNITE_RESOURCE_MEM_MB";
+
+ /** Memory limit. */
+ private double mem = UNLIMITED;
+
+ /** */
+ public static final String IGNITE_RESOURCE_DISK_MB = "IGNITE_RESOURCE_DISK_MB";
+
+ /** Disk space limit. */
+ private double disk = UNLIMITED;
+
+ /** */
+ public static final String IGNITE_RESOURCE_NODE_CNT = "IGNITE_RESOURCE_NODE_CNT";
+
+ /** Node count limit. */
+ private double nodeCnt = UNLIMITED;
+
+ /** */
+ public static final String IGNITE_RESOURCE_MIN_CPU_CNT_PER_NODE = "IGNITE_RESOURCE_MIN_CPU_CNT_PER_NODE";
+
+ /** */
+ public static final double DEFAULT_RESOURCE_MIN_CPU = 2;
+
+ /** Min memory per node. */
+ private double minCpu = DEFAULT_RESOURCE_MIN_CPU;
+
+ /** */
+ public static final String IGNITE_RESOURCE_MIN_MEMORY_PER_NODE = "IGNITE_RESOURCE_MIN_MEMORY_PER_NODE";
+
+ /** */
+ public static final double DEFAULT_RESOURCE_MIN_MEM = 256;
+
+ /** Min memory per node. */
+ private double minMemory = DEFAULT_RESOURCE_MIN_MEM;
+
+ /** */
+ public static final String IGNITE_VERSION = "IGNITE_VERSION";
+
+ /** */
+ public static final String DEFAULT_IGNITE_VERSION = "latest";
+
+ /** Ignite version. */
+ private String igniteVer = DEFAULT_IGNITE_VERSION;
+
+ /** */
+ public static final String IGNITE_WORK_DIR = "IGNITE_WORK_DIR";
+
+ /** */
+ public static final String DEFAULT_IGNITE_WORK_DIR = "~/ignite-releases";
+
+ /** Ignite version. */
+ private String igniteWorkDir = DEFAULT_IGNITE_WORK_DIR;
+
+ /** */
+ public static final String IGNITE_USERS_LIBS = "IGNITE_USERS_LIBS";
+
+ /** Path to users libs. */
+ private String userLibs = null;
+
+ /** */
+ public static final String IGNITE_CONFIG_XML = "IGNITE_XML_CONFIG";
+
+ /** Ignite config. */
+ private String igniteCfg = null;
+
+ /** */
+ public ClusterProperties() {
+ // No-op.
+ }
+
+ /**
+ * @return CPU count limit.
+ */
+ public double cpus(){
+ return cpu;
+ }
+
+ /**
+ * @return mem limit.
+ */
+ public double memory() {
+ return mem;
+ }
+
+ /**
+ * @return disk limit.
+ */
+ public double disk() {
+ return disk;
+ }
+
+ /**
+ * @return instance count limit.
+ */
+ public double instances() {
+ return nodeCnt;
+ }
+
+ /**
+ * @return min memory per node.
+ */
+ public double minMemoryPerNode() {
+ return minMemory;
+ }
+
+ /**
+ * @return min cpu count per node.
+ */
+ public double minCpuPerNode() {
+ return minCpu;
+ }
+
+ /**
+ * @return Ignite version.
+ */
+ public String igniteVer() {
+ return igniteVer;
+ }
+
+ /**
+ * @return Working directory.
+ */
+ public String igniteWorkDir() {
+ return igniteWorkDir;
+ }
+
+ /**
+ * @return User's libs.
+ */
+ public String userLibs() {
+ return userLibs;
+ }
+
+ /**
+ * @return Ignite configuration.
+ */
+ public String igniteCfg() {
+ return igniteCfg;
+ }
+
+ /**
+ * @return Master url.
+ */
+ public String masterUrl() {
+ return mesosUrl;
+ }
+
+ /**
+ * @param config path to config file.
+ * @return Cluster configuration.
+ */
+ public static ClusterProperties from(String config) {
+ try {
+ Properties props = null;
+
+ if (config != null) {
+ props = new Properties();
+
+ props.load(new FileInputStream(config));
+ }
+
+ ClusterProperties prop = new ClusterProperties();
+
+ prop.mesosUrl = getStringProperty(MESOS_MASTER_URL, props, DEFAULT_MESOS_MASTER_URL);
+
+ prop.cpu = getDoubleProperty(IGNITE_RESOURCE_CPU_CORES, props, UNLIMITED);
+ prop.mem = getDoubleProperty(IGNITE_RESOURCE_MEM_MB, props, UNLIMITED);
+ prop.disk = getDoubleProperty(IGNITE_RESOURCE_DISK_MB, props, UNLIMITED);
+ prop.nodeCnt = getDoubleProperty(IGNITE_RESOURCE_NODE_CNT, props, UNLIMITED);
+ prop.minCpu = getDoubleProperty(IGNITE_RESOURCE_MIN_CPU_CNT_PER_NODE, props, DEFAULT_RESOURCE_MIN_CPU);
+ prop.minMemory = getDoubleProperty(IGNITE_RESOURCE_MIN_MEMORY_PER_NODE, props, DEFAULT_RESOURCE_MIN_MEM);
+
+ prop.igniteVer = getStringProperty(IGNITE_VERSION, props, DEFAULT_IGNITE_VERSION);
+ prop.igniteWorkDir = getStringProperty(IGNITE_WORK_DIR, props, DEFAULT_IGNITE_WORK_DIR);
+ prop.igniteCfg = getStringProperty(IGNITE_CONFIG_XML, props, null);
+ prop.userLibs = getStringProperty(IGNITE_USERS_LIBS, props, null);
+
+ return prop;
+ }
+ catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ /**
+ * @param name Property name.
+ * @param fileProps Property file.
+ * @return Property value.
+ */
+ private static double getDoubleProperty(String name, Properties fileProps, Double defaultVal) {
+ if (fileProps != null && fileProps.containsKey(name))
+ return Double.valueOf(fileProps.getProperty(name));
+
+ String property = System.getProperty(name);
+
+ if (property == null)
+ property = System.getenv(name);
+
+ return property == null ? defaultVal : Double.valueOf(property);
+ }
+
+ /**
+ * @param name Property name.
+ * @param fileProps Property file.
+ * @return Property value.
+ */
+ private static String getStringProperty(String name, Properties fileProps, String defaultVal) {
+ if (fileProps != null && fileProps.containsKey(name))
+ return fileProps.getProperty(name);
+
+ String property = System.getProperty(name);
+
+ if (property == null)
+ property = System.getenv(name);
+
+ return property == null ? defaultVal : property;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/4b482187/modules/mesos/src/main/java/org/apache/ignite/mesos/ClusterResources.java
----------------------------------------------------------------------
diff --git a/modules/mesos/src/main/java/org/apache/ignite/mesos/ClusterResources.java b/modules/mesos/src/main/java/org/apache/ignite/mesos/ClusterResources.java
deleted file mode 100644
index 1887530..0000000
--- a/modules/mesos/src/main/java/org/apache/ignite/mesos/ClusterResources.java
+++ /dev/null
@@ -1,160 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.mesos;
-
-import java.io.*;
-import java.util.*;
-
-/**
- * Cluster settings.
- */
-public class ClusterResources {
- /** Unlimited. */
- public static final int DEFAULT_VALUE = -1;
-
- /** */
- public static final String IGNITE_RESOURCE_CPU_CORES = "IGNITE_RESOURCE_CPU_CORES";
-
- /** CPU limit. */
- private double cpu = DEFAULT_VALUE;
-
- /** */
- public static final String IGNITE_RESOURCE_MEM_MB = "IGNITE_RESOURCE_MEM_MB";
-
- /** Memory limit. */
- private double mem = DEFAULT_VALUE;
-
- /** */
- public static final String IGNITE_RESOURCE_DISK_MB = "IGNITE_RESOURCE_DISK_MB";
-
- /** Disk space limit. */
- private double disk = DEFAULT_VALUE;
-
- /** */
- public static final String IGNITE_RESOURCE_NODE_CNT = "IGNITE_RESOURCE_NODE_CNT";
-
- /** Node count limit. */
- private double nodeCnt = DEFAULT_VALUE;
-
- /** */
- public static final String IGNITE_RESOURCE_MIN_CPU_CNT_PER_NODE = "IGNITE_RESOURCE_MIN_CPU_CNT_PER_NODE";
-
- /** Min memory per node. */
- private int minCpu = 2;
-
- /** */
- public static final String IGNITE_RESOURCE_MIN_MEMORY_PER_NODE = "IGNITE_RESOURCE_MIN_MEMORY_PER_NODE";
-
- /** Min memory per node. */
- private int minMemoryCnt = 256;
-
- /** */
- public ClusterResources() {
- // No-op.
- }
-
- /**
- * @return CPU count limit.
- */
- public double cpus(){
- return cpu;
- }
-
- /**
- * @return mem limit.
- */
- public double memory() {
- return mem;
- }
-
- /**
- * @return disk limit.
- */
- public double disk() {
- return disk;
- }
-
- /**
- * @return instance count limit.
- */
- public double instances() {
- return nodeCnt;
- }
-
- /**
- * @return min memory per node.
- */
- public int minMemoryPerNode() {
- return minMemoryCnt;
- }
-
- /**
- * @return min cpu count per node.
- */
- public int minCpuPerNode() {
- return minCpu;
- }
-
- /**
- * @param config path to config file.
- * @return Cluster configuration.
- */
- public static ClusterResources from(String config) {
- try {
- Properties props = null;
-
- if (config != null) {
- props = new Properties();
-
- props.load(new FileInputStream(config));
- }
-
- ClusterResources resources = new ClusterResources();
-
- resources.cpu = getProperty(IGNITE_RESOURCE_CPU_CORES, props);
- resources.mem = getProperty(IGNITE_RESOURCE_MEM_MB, props);
- resources.disk = getProperty(IGNITE_RESOURCE_DISK_MB, props);
- resources.nodeCnt = getProperty(IGNITE_RESOURCE_NODE_CNT, props);
-
- return resources;
- }
- catch (IOException e) {
- throw new RuntimeException(e);
- }
- }
-
- /**
- * @param name Property name.
- * @param fileProps Property file.
- * @return Property value.
- */
- private static double getProperty(String name, Properties fileProps) {
- if (fileProps != null && fileProps.containsKey(name))
- return Double.valueOf(fileProps.getProperty(name));
-
- String property = System.getProperty(name);
-
- if (property == null)
- property = System.getenv(name);
-
- if (property == null)
- return DEFAULT_VALUE;
-
- return Double.valueOf(property);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/4b482187/modules/mesos/src/main/java/org/apache/ignite/mesos/IgniteFramework.java
----------------------------------------------------------------------
diff --git a/modules/mesos/src/main/java/org/apache/ignite/mesos/IgniteFramework.java b/modules/mesos/src/main/java/org/apache/ignite/mesos/IgniteFramework.java
index 2d74f71..0ff945b 100644
--- a/modules/mesos/src/main/java/org/apache/ignite/mesos/IgniteFramework.java
+++ b/modules/mesos/src/main/java/org/apache/ignite/mesos/IgniteFramework.java
@@ -18,22 +18,29 @@
package org.apache.ignite.mesos;
import com.google.protobuf.*;
+import org.apache.ignite.mesos.resource.*;
import org.apache.mesos.*;
+import org.glassfish.grizzly.http.server.*;
+import org.glassfish.jersey.grizzly2.httpserver.*;
+import org.glassfish.jersey.server.*;
+
+import java.net.*;
/**
* TODO
*/
public class IgniteFramework {
+
+ public static final String IGNITE_FRAMEWORK_NAME = "IgniteFramework";
+
/**
- * @param args Args [host:port] [resource limit]
+ * @param args Args
*/
- public static void main(String[] args) {
- checkArgs(args);
-
+ public static void main(String[] args) throws Exception {
final int frameworkFailoverTimeout = 0;
Protos.FrameworkInfo.Builder frameworkBuilder = Protos.FrameworkInfo.newBuilder()
- .setName("IgniteFramework")
+ .setName(IGNITE_FRAMEWORK_NAME)
.setUser("") // Have Mesos fill in the current user.
.setFailoverTimeout(frameworkFailoverTimeout); // timeout in seconds
@@ -42,8 +49,26 @@ public class IgniteFramework {
frameworkBuilder.setCheckpoint(true);
}
+ ClusterProperties clusterProperties = ClusterProperties.from(args.length == 1 ? args[0] : null);
+
+ String baseUrl = String.format("http://%s:%d", formatInetAddress(InetAddress.getLocalHost()), 4444);
+
+ URI httpServerBaseUri = URI.create(baseUrl);
+
+ ResourceConfig rc = new ResourceConfig()
+ .registerInstances(new ResourceController(clusterProperties.userLibs(), clusterProperties.igniteCfg(),
+ clusterProperties.igniteWorkDir()));
+
+ HttpServer httpServer = GrizzlyHttpServerFactory.createHttpServer(httpServerBaseUri, rc);
+
+ ResourceProvider provider = new ResourceProvider();
+
+ IgniteProvider igniteProvider = new IgniteProvider(clusterProperties.igniteWorkDir());
+
+ provider.init(clusterProperties, igniteProvider, baseUrl);
+
// Create the scheduler.
- final Scheduler scheduler = new IgniteScheduler(ClusterResources.from(null));
+ Scheduler scheduler = new IgniteScheduler(clusterProperties, provider);
// create the driver
MesosSchedulerDriver driver;
@@ -67,31 +92,34 @@ public class IgniteFramework {
frameworkBuilder.setPrincipal(System.getenv("DEFAULT_PRINCIPAL"));
- driver = new MesosSchedulerDriver(scheduler, frameworkBuilder.build(), args[0], credential);
+ driver = new MesosSchedulerDriver(scheduler, frameworkBuilder.build(), clusterProperties.masterUrl(),
+ credential);
}
else {
frameworkBuilder.setPrincipal("ignite-framework-java");
- driver = new MesosSchedulerDriver(scheduler, frameworkBuilder.build(), args[0]);
+ driver = new MesosSchedulerDriver(scheduler, frameworkBuilder.build(), clusterProperties.masterUrl());
}
int status = driver.run() == Protos.Status.DRIVER_STOPPED ? 0 : 1;
+ httpServer.shutdown();
+
// Ensure that the driver process terminates.
driver.stop();
System.exit(status);
}
- /**
- * Check input arguments.
- *
- * @param args Arguments.
- */
- private static void checkArgs(String[] args) {
- if (args.length == 0)
- throw new IllegalArgumentException("Illegal arguments.");
-
- // TODO: add more
+ public static String formatInetAddress(final InetAddress inetAddress) {
+ if (inetAddress instanceof Inet4Address) {
+ return inetAddress.getHostAddress();
+ }
+ else if (inetAddress instanceof Inet6Address) {
+ return String.format("[%s]", inetAddress.getHostAddress());
+ }
+ else
+ throw new IllegalArgumentException("InetAddress type: " + inetAddress.getClass().getName() +
+ " is not supported");
}
}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/4b482187/modules/mesos/src/main/java/org/apache/ignite/mesos/IgniteScheduler.java
----------------------------------------------------------------------
diff --git a/modules/mesos/src/main/java/org/apache/ignite/mesos/IgniteScheduler.java b/modules/mesos/src/main/java/org/apache/ignite/mesos/IgniteScheduler.java
index 9d10860..b1ff930 100644
--- a/modules/mesos/src/main/java/org/apache/ignite/mesos/IgniteScheduler.java
+++ b/modules/mesos/src/main/java/org/apache/ignite/mesos/IgniteScheduler.java
@@ -17,6 +17,7 @@
package org.apache.ignite.mesos;
+import org.apache.ignite.mesos.resource.*;
import org.apache.mesos.*;
import org.slf4j.*;
@@ -27,12 +28,6 @@ import java.util.concurrent.atomic.*;
* TODO
*/
public class IgniteScheduler implements Scheduler {
- /** Docker image name. */
- public static final String IMAGE = "apacheignite/ignite-docker";
-
- /** Startup sctipt path. */
- public static final String STARTUP_SCRIPT = "/home/ignite/startup.sh";
-
/** Cpus. */
public static final String CPUS = "cpus";
@@ -61,13 +56,18 @@ public class IgniteScheduler implements Scheduler {
private Map<String, IgniteTask> tasks = new HashMap<>();
/** Cluster resources. */
- private ClusterResources clusterLimit;
+ private ClusterProperties clusterLimit;
+
+ /** Resource provider. */
+ private ResourceProvider resourceProvider;
/**
- * @param clusterLimit Resources limit.
+ * @param clusterLimit Cluster limit.
+ * @param resourceProvider Resource provider.
*/
- public IgniteScheduler(ClusterResources clusterLimit) {
+ public IgniteScheduler(ClusterProperties clusterLimit, ResourceProvider resourceProvider) {
this.clusterLimit = clusterLimit;
+ this.resourceProvider = resourceProvider;
}
/** {@inheritDoc} */
@@ -114,6 +114,7 @@ public class IgniteScheduler implements Scheduler {
}
}
+
/**
* Create Task.
*
@@ -123,20 +124,36 @@ public class IgniteScheduler implements Scheduler {
* @return Task.
*/
protected Protos.TaskInfo createTask(Protos.Offer offer, IgniteTask igniteTask, Protos.TaskID taskId) {
- // Docker image info.
- Protos.ContainerInfo.DockerInfo.Builder docker = Protos.ContainerInfo.DockerInfo.newBuilder()
- .setImage(IMAGE)
- .setNetwork(Protos.ContainerInfo.DockerInfo.Network.HOST);
-
- // Container info.
- Protos.ContainerInfo.Builder cont = Protos.ContainerInfo.newBuilder();
- cont.setType(Protos.ContainerInfo.Type.DOCKER);
- cont.setDocker(docker.build());
+ Protos.CommandInfo.Builder builder = Protos.CommandInfo.newBuilder()
+ .setEnvironment(Protos.Environment.newBuilder().addVariables(Protos.Environment.Variable.newBuilder()
+ .setName("IGNITE_TCP_DISCOVERY_ADDRESSES")
+ .setValue(getAddress())))
+ .addUris(Protos.CommandInfo.URI.newBuilder()
+ .setValue(resourceProvider.igniteUrl())
+ .setExtract(true))
+ .addUris(Protos.CommandInfo.URI.newBuilder().setValue(resourceProvider.igniteConfigUrl()));
+
+ if (resourceProvider.resourceUrl() != null) {
+ for (String url : resourceProvider.resourceUrl())
+ builder.addUris(Protos.CommandInfo.URI.newBuilder().setValue(url));
+
+ builder.setValue("cp *.jar ./gridgain-community-*/libs/ "
+ + "&& ./gridgain-community-*/bin/ignite.sh "
+ + resourceProvider.configName()
+ + " -J-Xmx" + String.valueOf((int) igniteTask.mem() + "m")
+ + " -J-Xms" + String.valueOf((int) igniteTask.mem()) + "m");
+ }
+ else
+ builder.setValue("./gridgain-community-*/bin/ignite.sh "
+ + resourceProvider.configName()
+ + " -J-Xmx" + String.valueOf((int) igniteTask.mem() + "m")
+ + " -J-Xms" + String.valueOf((int) igniteTask.mem()) + "m");
return Protos.TaskInfo.newBuilder()
.setName("Ignite node " + taskId.getValue())
.setTaskId(taskId)
.setSlaveId(offer.getSlaveId())
+ .setCommand(builder)
.addResources(Protos.Resource.newBuilder()
.setName(CPUS)
.setType(Protos.Value.Type.SCALAR)
@@ -145,12 +162,6 @@ public class IgniteScheduler implements Scheduler {
.setName(MEM)
.setType(Protos.Value.Type.SCALAR)
.setScalar(Protos.Value.Scalar.newBuilder().setValue(igniteTask.mem())))
- .setContainer(cont)
- .setCommand(Protos.CommandInfo.newBuilder()
- .setShell(false)
- .addArguments(STARTUP_SCRIPT)
- .addArguments(String.valueOf((int) igniteTask.mem()))
- .addArguments(getAddress()))
.build();
}
@@ -227,11 +238,11 @@ public class IgniteScheduler implements Scheduler {
totalDisk += task.disk();
}
- cpus = clusterLimit.cpus() == ClusterResources.DEFAULT_VALUE ? cpus :
+ cpus = clusterLimit.cpus() == ClusterProperties.UNLIMITED ? cpus :
Math.min(clusterLimit.cpus() - totalCpus, cpus);
- mem = clusterLimit.memory() == ClusterResources.DEFAULT_VALUE ? mem :
+ mem = clusterLimit.memory() == ClusterProperties.UNLIMITED ? mem :
Math.min(clusterLimit.memory() - totalMem, mem);
- disk = clusterLimit.disk() == ClusterResources.DEFAULT_VALUE ? disk :
+ disk = clusterLimit.disk() == ClusterProperties.UNLIMITED ? disk :
Math.min(clusterLimit.disk() - totalDisk, disk);
if (cpus > 0 && mem > 0)
@@ -253,7 +264,7 @@ public class IgniteScheduler implements Scheduler {
* @return {@code True} if limit isn't violated else {@code false}.
*/
private boolean checkLimit(double limit, double value) {
- return limit == ClusterResources.DEFAULT_VALUE || limit <= value;
+ return limit == ClusterProperties.UNLIMITED || limit <= value;
}
/** {@inheritDoc} */
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/4b482187/modules/mesos/src/main/java/org/apache/ignite/mesos/resource/IgniteProvider.java
----------------------------------------------------------------------
diff --git a/modules/mesos/src/main/java/org/apache/ignite/mesos/resource/IgniteProvider.java b/modules/mesos/src/main/java/org/apache/ignite/mesos/resource/IgniteProvider.java
new file mode 100644
index 0000000..18ceb00
--- /dev/null
+++ b/modules/mesos/src/main/java/org/apache/ignite/mesos/resource/IgniteProvider.java
@@ -0,0 +1,234 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.mesos.resource;
+
+import java.io.*;
+import java.net.*;
+import java.nio.channels.*;
+import java.util.*;
+
+/**
+ * TODO
+ */
+public class IgniteProvider {
+ /** */
+ public static final String DOWNLOAD_LINK = "http://tiny.cc/updater/download_community.php";
+
+ /** */
+ public static final String DIRECT_DOWNLOAD_LINK = "http://www.gridgain.com/media/gridgain-community-fabric-";
+
+ /** */
+ private String downloadFolder;
+
+ /** */
+ private String latestVersion = null;
+
+ /**
+ * @param downloadFolder Folder with ignite.
+ */
+ public IgniteProvider(String downloadFolder) {
+ this.downloadFolder = downloadFolder;
+ }
+
+ /**
+ * @return Latest ignite version.
+ */
+ public String getIgnite() {
+ File folder = checkDownloadFolder();
+
+ if (latestVersion == null) {
+ List<String> files = findIgnites(folder);
+
+ if (!files.isEmpty()) {
+ if (files.size() == 1)
+ latestVersion = parseVersion(files.get(0));
+ else
+ latestVersion = parseVersion(Collections.max(files, new Comparator<String>() {
+ @Override public int compare(String f1, String f2) {
+ if (f1.equals(f2))
+ return 0;
+
+ String[] ver1 = parseVersion(f1).split("\\.");
+ String[] ver2 = parseVersion(f2).split("\\.");
+
+ if (Integer.valueOf(ver1[0]) > Integer.valueOf(ver2[0])
+ && Integer.valueOf(ver1[1]) > Integer.valueOf(ver2[1])
+ && Integer.valueOf(ver1[2]) > Integer.valueOf(ver2[2]))
+
+ return 1;
+ else
+ return -1;
+ }
+ }));
+ }
+ }
+
+ latestVersion = updateIgnite(latestVersion);
+
+ return "gridgain-community-fabric-" + latestVersion + ".zip";
+ }
+
+ /**
+ * @param folder Folder.
+ * @return Ignite archives.
+ */
+ private List<String> findIgnites(File folder) {
+ String[] files = folder.list();
+
+ List<String> ignites = new ArrayList<>();
+
+ if (files != null) {
+ for (String fileName : files) {
+ if (fileName.contains("gridgain-community-fabric-") && fileName.endsWith(".zip"))
+ ignites.add(fileName);
+ }
+ }
+
+ return ignites;
+ }
+
+ /**
+ * @param version Ignite version.
+ * @return Ignite.
+ */
+ public String getIgnite(String version) {
+ File folder = checkDownloadFolder();
+
+ String[] ignites = folder.list();
+
+ String ignite = null;
+
+ if (ignites != null) {
+ for (String fileName : ignites) {
+ if (fileName.equals("gridgain-community-fabric-" + version + ".zip"))
+ ignite = fileName;
+ }
+ }
+
+ if (ignite != null)
+ return ignite;
+
+ return downloadIgnite(version);
+ }
+
+ /**
+ * @param currentVersion The current latest version.
+ * @return Current version if the current version is latest; new ignite version otherwise.
+ */
+ private String updateIgnite(String currentVersion) {
+ try {
+ URL url;
+
+ if (currentVersion == null)
+ url = new URL(DOWNLOAD_LINK);
+ else
+ url = new URL(DOWNLOAD_LINK + "?version=" + currentVersion);
+
+ HttpURLConnection conn = (HttpURLConnection)url.openConnection();
+
+ int code = conn.getResponseCode();
+
+ if (code == 200) {
+ String redirectUrl = conn.getURL().toString();
+
+ checkDownloadFolder();
+
+ FileOutputStream outFile = new FileOutputStream(downloadFolder + "/" + fileName(redirectUrl));
+
+ outFile.getChannel().transferFrom(Channels.newChannel(conn.getInputStream()), 0, Long.MAX_VALUE);
+
+ outFile.close();
+
+ return parseVersion(redirectUrl);
+ } else if (code == 304)
+ // This version is latest.
+ return currentVersion;
+ else
+ throw new RuntimeException("Got unexpected response code. Response code: " + code);
+ }
+ catch (IOException e) {
+ throw new RuntimeException("Failed update ignite.", e);
+ }
+ }
+
+ /**
+ * @param version The current latest version.
+ * @return Ignite archive.
+ */
+ public String downloadIgnite(String version) {
+ try {
+ URL url = new URL(DIRECT_DOWNLOAD_LINK + version + ".zip");
+
+ HttpURLConnection conn = (HttpURLConnection)url.openConnection();
+
+ int code = conn.getResponseCode();
+
+ if (code == 200) {
+ checkDownloadFolder();
+
+ String fileName = fileName(url.toString());
+
+ FileOutputStream outFile = new FileOutputStream(downloadFolder + fileName);
+
+ outFile.getChannel().transferFrom(Channels.newChannel(conn.getInputStream()), 0, Long.MAX_VALUE);
+
+ outFile.close();
+
+ return fileName;
+ }
+ else
+ throw new RuntimeException("Got unexpected response code. Response code: " + code);
+ }
+ catch (IOException e) {
+ throw new RuntimeException("Failed update ignite.", e);
+ }
+ }
+
+
+ /**
+ * @return Download folder.
+ */
+ private File checkDownloadFolder() {
+ File file = new File(downloadFolder);
+
+ if (!file.exists())
+ file.mkdirs();
+
+ return file;
+ }
+
+ /**
+ * @param url URL.
+ * @return Ignite version.
+ */
+ public static String parseVersion(String url) {
+ String[] split = url.split("-");
+
+ return split[split.length - 1].replaceAll(".zip", "");
+ }
+
+ /**
+ * @param url URL.
+ * @return File name.
+ */
+ private static String fileName(String url) {
+ String[] split = url.split("/");
+
+ return split[split.length - 1];
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/4b482187/modules/mesos/src/main/java/org/apache/ignite/mesos/resource/ResourceController.java
----------------------------------------------------------------------
diff --git a/modules/mesos/src/main/java/org/apache/ignite/mesos/resource/ResourceController.java b/modules/mesos/src/main/java/org/apache/ignite/mesos/resource/ResourceController.java
new file mode 100644
index 0000000..5c9e693
--- /dev/null
+++ b/modules/mesos/src/main/java/org/apache/ignite/mesos/resource/ResourceController.java
@@ -0,0 +1,130 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.mesos.resource;
+
+import javax.ws.rs.*;
+import javax.ws.rs.core.*;
+import java.io.*;
+
+/**
+ *
+ */
+@Path("/")
+public class ResourceController {
+ /** */
+ public static final String IGNITE_PREFIX = "/ignite/";
+
+ /** */
+ public static final String LIBS_PREFIX = "/libs/";
+
+ /** */
+ public static final String CONFIG_PREFIX = "/config/";
+
+ /** */
+ public static final String DEFAULT_CONFIG = CONFIG_PREFIX + "default/";
+
+ /** */
+ private String libsDir;
+
+ /** */
+ private String cfgPath;
+
+ /** */
+ private String igniteDir;
+
+ /**
+ * @param libsDir Path to directory with user libs.
+ * @param cfgPath Path to config file.
+ */
+ public ResourceController(String libsDir, String cfgPath, String igniteDir) {
+ this.libsDir = libsDir;
+ this.cfgPath = cfgPath;
+ this.igniteDir = igniteDir;
+ }
+
+ /**
+ *
+ * @param ignite
+ * @return
+ */
+ @GET
+ @Path(IGNITE_PREFIX + "{ignite-dist}")
+ public Response ignite(@PathParam("ignite-dist") String ignite) {
+ return handleRequest(new File(igniteDir + "/" + ignite), "application/zip-archive", ignite);
+ }
+
+ /**
+ *
+ * @param lib
+ * @return
+ */
+ @GET
+ @Path(LIBS_PREFIX + "{lib}")
+ public Response lib(@PathParam("lib") String lib) {
+ return handleRequest(new File(libsDir + "/" + lib), "application/java-archive", lib);
+ }
+
+ /**
+ *
+ * @param cfg
+ * @return
+ */
+ @GET
+ @Path(CONFIG_PREFIX + "{cfg}")
+ public Response config(@PathParam("cfg") String cfg) {
+ return handleRequest(new File(cfgPath), "application/xml", cfg);
+ }
+
+ /**
+ *
+ * @param cfg
+ * @return
+ */
+ @GET
+ @Path(DEFAULT_CONFIG + "{cfg}")
+ public Response defaultConfig(@PathParam("cfg") String cfg) {
+ return handleRequest(Thread.currentThread().getContextClassLoader().getResourceAsStream(cfg),
+ "application/xml", cfg);
+ }
+
+ /**
+ *
+ * @param resource File resource.
+ * @param type Type.
+ * @param attachmentName Attachment name.
+ * @return Http response.
+ */
+ private static Response handleRequest(File resource, String type, String attachmentName) {
+ final Response.ResponseBuilder builder = Response.ok(resource, type);
+ builder.header("Content-Disposition", String.format("attachment; filename=\"%s\"", attachmentName));
+ return builder.build();
+ }
+
+ /**
+ *
+ * @param resource File resource.
+ * @param type Type.
+ * @param attachmentName Attachment name.
+ * @return Http response.
+ */
+ private static Response handleRequest(InputStream resource, String type, String attachmentName) {
+ final Response.ResponseBuilder builder = Response.ok(resource, type);
+ builder.header("Content-Disposition", String.format("attachment; filename=\"%s\"", attachmentName));
+ return builder.build();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/4b482187/modules/mesos/src/main/java/org/apache/ignite/mesos/resource/ResourceProvider.java
----------------------------------------------------------------------
diff --git a/modules/mesos/src/main/java/org/apache/ignite/mesos/resource/ResourceProvider.java b/modules/mesos/src/main/java/org/apache/ignite/mesos/resource/ResourceProvider.java
new file mode 100644
index 0000000..544b15c
--- /dev/null
+++ b/modules/mesos/src/main/java/org/apache/ignite/mesos/resource/ResourceProvider.java
@@ -0,0 +1,120 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.mesos.resource;
+
+import org.apache.ignite.mesos.*;
+
+import java.io.*;
+import java.util.*;
+
+import static org.apache.ignite.mesos.resource.ResourceController.*;
+
+/**
+ *
+ */
+public class ResourceProvider {
+ /** Ignite url. */
+ private String igniteUrl;
+
+ /** Resources. */
+ private Collection<String> libsUris;
+
+ /** Url config. */
+ private String configUrl;
+
+ /** Config name. */
+ private String configName;
+
+ /**
+ * @param properties Cluster properties.
+ * @param provider Ignite provider.
+ * @param baseUrl Base url.
+ */
+ public void init(ClusterProperties properties, IgniteProvider provider, String baseUrl) {
+ // Downloading ignite.
+ if (properties.igniteVer().equals(ClusterProperties.DEFAULT_IGNITE_VERSION))
+ igniteUrl = baseUrl + IGNITE_PREFIX + provider.getIgnite();
+ else
+ igniteUrl = baseUrl + IGNITE_PREFIX + provider.getIgnite(properties.igniteVer());
+
+ // Find all jar files into user folder.
+ if (properties.userLibs() != null && !properties.userLibs().isEmpty()) {
+ File libsDir = new File(properties.userLibs());
+
+ List<String> libs = new ArrayList<>();
+
+ if (libsDir.isDirectory()) {
+ File[] files = libsDir.listFiles();
+
+ if (files != null) {
+ for (File lib : files) {
+ if (lib.isFile() && lib.canRead() &&
+ (lib.getName().endsWith(".jar") || lib.getName().endsWith(".JAR")))
+ libs.add(baseUrl + LIBS_PREFIX + lib.getName());
+ }
+ }
+ }
+
+ libsUris = libs.isEmpty() ? null : libs;
+ }
+
+ // Set configuration url.
+ if (properties.igniteCfg() != null) {
+ File cfg = new File(properties.igniteCfg());
+
+ if (cfg.isFile() && cfg.canRead()) {
+ configUrl = baseUrl + CONFIG_PREFIX + cfg.getName();
+
+ configName = cfg.getName();
+ }
+ }
+ else {
+ configName = "ignite-default-config.xml";
+
+ configUrl = baseUrl + DEFAULT_CONFIG + configName;
+ }
+ }
+
+ /**
+ * @return Config name.
+ */
+ public String configName() {
+ return configName;
+ }
+
+ /**
+ * @return Ignite url.
+ */
+ public String igniteUrl() {
+ return igniteUrl;
+ }
+
+ /**
+ * @return Urls to user's libs.
+ */
+ public Collection<String> resourceUrl() {
+ return libsUris;
+ }
+
+ /**
+ * @return Url to config file.
+ */
+ public String igniteConfigUrl() {
+ return configUrl;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/4b482187/modules/mesos/src/main/resources/ignite-default-config.xml
----------------------------------------------------------------------
diff --git a/modules/mesos/src/main/resources/ignite-default-config.xml b/modules/mesos/src/main/resources/ignite-default-config.xml
new file mode 100644
index 0000000..9fcce97
--- /dev/null
+++ b/modules/mesos/src/main/resources/ignite-default-config.xml
@@ -0,0 +1,35 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<beans xmlns="http://www.springframework.org/schema/beans"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://www.springframework.org/schema/beans
+ http://www.springframework.org/schema/beans/spring-beans.xsd">
+ <bean class="org.apache.ignite.configuration.IgniteConfiguration">
+ <property name="discoverySpi">
+ <bean class="org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi">
+ <property name="ipFinder">
+ <bean class="org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder">
+ <property name="shared" value="true"/>
+ </bean>
+ </property>
+ </bean>
+ </property>
+ </bean>
+</beans>
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/4b482187/modules/mesos/src/main/resources/log4j2.xml
----------------------------------------------------------------------
diff --git a/modules/mesos/src/main/resources/log4j2.xml b/modules/mesos/src/main/resources/log4j2.xml
new file mode 100644
index 0000000..d66a83f
--- /dev/null
+++ b/modules/mesos/src/main/resources/log4j2.xml
@@ -0,0 +1,35 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<configuration strict="true" status="OFF" monitorInterval="30" shutdownHook="disable">
+
+ <appenders>
+ <appender type="Console" name="STDOUT">
+ <layout type="PatternLayout" pattern="%d %p %C{1.} [%t] %m%n %ex"/>
+ </appender>
+ </appenders>
+
+ <loggers>
+ <root level="info">
+ <appender-ref ref="STDOUT"/>
+ </root>
+
+ </loggers>
+
+</configuration>
\ No newline at end of file