You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@heron.apache.org by nw...@apache.org on 2019/01/22 23:08:33 UTC
[incubator-heron] branch master updated: create a symbolic link for
kubernetes conf after loading docker image (#3161)
This is an automated email from the ASF dual-hosted git repository.
nwang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-heron.git
The following commit(s) were added to refs/heads/master by this push:
new 8ac36a2 create a symbolic link for kubernetes conf after loading docker image (#3161)
8ac36a2 is described below
commit 8ac36a24bf083bf0cc718431ff3bb427dc7228b0
Author: Yao Li <cl...@gmail.com>
AuthorDate: Tue Jan 22 15:08:29 2019 -0800
create a symbolic link for kubernetes conf after loading docker image (#3161)
* create a symbolic link for kubernetes conf after loading docker image
* address comment: remove platform specific K-V in spi config
---
heron/downloaders/src/shell/BUILD | 5 +++++
.../src/shell/heron-downloader-config.sh | 22 ++++++++++++++++++++++
.../kubernetes/AppsV1beta1Controller.java | 3 ++-
.../scheduler/kubernetes/KubernetesUtils.java | 6 ++++++
.../java/org/apache/heron/spi/common/Context.java | 4 ++++
.../src/java/org/apache/heron/spi/common/Key.java | 1 +
scripts/packages/BUILD | 2 ++
7 files changed, 42 insertions(+), 1 deletion(-)
diff --git a/heron/downloaders/src/shell/BUILD b/heron/downloaders/src/shell/BUILD
index 9e8f783..9af9722 100644
--- a/heron/downloaders/src/shell/BUILD
+++ b/heron/downloaders/src/shell/BUILD
@@ -4,3 +4,8 @@ sh_binary(
name = "heron-downloader",
srcs = ["heron-downloader.sh"],
)
+
+sh_binary(
+ name = "heron-downloader-config",
+ srcs = ["heron-downloader-config.sh"],
+)
diff --git a/heron/downloaders/src/shell/heron-downloader-config.sh b/heron/downloaders/src/shell/heron-downloader-config.sh
new file mode 100755
index 0000000..07a0dea
--- /dev/null
+++ b/heron/downloaders/src/shell/heron-downloader-config.sh
@@ -0,0 +1,22 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+if [ $1 = "kubernetes" ]; then
+ ln -s /usr/local/heron/conf/kubernetes /heron
+ mv /heron/kubernetes /heron/heron-conf
+fi
diff --git a/heron/schedulers/src/java/org/apache/heron/scheduler/kubernetes/AppsV1beta1Controller.java b/heron/schedulers/src/java/org/apache/heron/scheduler/kubernetes/AppsV1beta1Controller.java
index 4a16adc..2aea58d 100644
--- a/heron/schedulers/src/java/org/apache/heron/scheduler/kubernetes/AppsV1beta1Controller.java
+++ b/heron/schedulers/src/java/org/apache/heron/scheduler/kubernetes/AppsV1beta1Controller.java
@@ -257,7 +257,8 @@ public class AppsV1beta1Controller extends KubernetesController {
return Arrays.asList(
"sh",
"-c",
- KubernetesUtils.getFetchCommand(configuration, runtimeConfiguration)
+ KubernetesUtils.getConfCommand(configuration)
+ + " && " + KubernetesUtils.getFetchCommand(configuration, runtimeConfiguration)
+ " && " + setShardIdEnvironmentVariableCommand()
+ " && " + String.join(" ", executorCommand)
);
diff --git a/heron/schedulers/src/java/org/apache/heron/scheduler/kubernetes/KubernetesUtils.java b/heron/schedulers/src/java/org/apache/heron/scheduler/kubernetes/KubernetesUtils.java
index b5e3b6c..76f432a 100644
--- a/heron/schedulers/src/java/org/apache/heron/scheduler/kubernetes/KubernetesUtils.java
+++ b/heron/schedulers/src/java/org/apache/heron/scheduler/kubernetes/KubernetesUtils.java
@@ -35,9 +35,15 @@ import io.kubernetes.client.ApiException;
final class KubernetesUtils {
+ private static final String CONTAINER = "kubernetes";
+
private KubernetesUtils() {
}
+ static String getConfCommand(Config config) {
+ return String.format("%s %s", Context.downloaderConfKubernetes(config), CONTAINER);
+ }
+
static String getFetchCommand(Config config, Config runtime) {
return String.format("%s %s .", Context.downloaderBinary(config),
Runtime.topologyPackageUri(runtime).toString());
diff --git a/heron/spi/src/java/org/apache/heron/spi/common/Context.java b/heron/spi/src/java/org/apache/heron/spi/common/Context.java
index d3bb3a1..81dfeab 100644
--- a/heron/spi/src/java/org/apache/heron/spi/common/Context.java
+++ b/heron/spi/src/java/org/apache/heron/spi/common/Context.java
@@ -346,6 +346,10 @@ public class Context {
return cfg.getStringValue(Key.DOWNLOADER_BINARY);
}
+ public static String downloaderConfKubernetes(Config cfg) {
+ return cfg.getStringValue(Key.DOWNLOADER_CONF);
+ }
+
public static String updatePrompt(Config cfg) {
return cfg.getStringValue(Key.UPDATE_PROMPT);
}
diff --git a/heron/spi/src/java/org/apache/heron/spi/common/Key.java b/heron/spi/src/java/org/apache/heron/spi/common/Key.java
index c216bf4..db2bd96 100644
--- a/heron/spi/src/java/org/apache/heron/spi/common/Key.java
+++ b/heron/spi/src/java/org/apache/heron/spi/common/Key.java
@@ -187,6 +187,7 @@ public enum Key {
PYTHON_INSTANCE_BINARY("heron.binaries.python.instance", "${HERON_BIN}/heron-python-instance"),
CPP_INSTANCE_BINARY ("heron.binaries.cpp.instance", "${HERON_BIN}/heron-cpp-instance"),
DOWNLOADER_BINARY ("heron.binaries.downloader", "${HERON_BIN}/heron-downloader"),
+ DOWNLOADER_CONF ("heron.binaries.downloader-conf", "${HERON_BIN}/heron-downloader-config"),
// keys for `heron` command line.
// Prompt user when more containers are required so that
diff --git a/scripts/packages/BUILD b/scripts/packages/BUILD
index ae67e49..2abf998 100644
--- a/scripts/packages/BUILD
+++ b/scripts/packages/BUILD
@@ -73,6 +73,7 @@ pkg_tar(
srcs = [
"//heron/executor/src/python:heron-executor",
"//heron/downloaders/src/shell:heron-downloader",
+ "//heron/downloaders/src/shell:heron-downloader-config",
"//heron/instance/src/python:heron-python-instance",
"//heron/instance/src/cpp:heron-cpp-instance",
"//heron/shell/src/python:heron-shell",
@@ -264,6 +265,7 @@ pkg_tar(
package_dir = "bin",
srcs = [
"//heron/downloaders/src/shell:heron-downloader",
+ "//heron/downloaders/src/shell:heron-downloader-config",
"//heron/tools/cli/src/python:heron",
"//heron/tools/explorer/src/python:heron-explorer",
"//heron/tools/admin/src/python:heron-admin",