You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@heron.apache.org by hu...@apache.org on 2018/05/02 21:05:45 UTC
[incubator-heron] branch huijunw/refactordownloader created (now
cf976de)
This is an automated email from the ASF dual-hosted git repository.
huijun pushed a change to branch huijunw/refactordownloader
in repository https://gitbox.apache.org/repos/asf/incubator-heron.git.
at cf976de move downloader registry map to yaml
This branch includes the following new commits:
new cf976de move downloader registry map to yaml
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
--
To stop receiving notification emails like this one, please contact
huijun@apache.org.
[incubator-heron] 01/01: move downloader registry map to yaml
Posted by hu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
huijun pushed a commit to branch huijunw/refactordownloader
in repository https://gitbox.apache.org/repos/asf/incubator-heron.git
commit cf976de5eec7ababe398a45a351247277a0c6648
Author: Huijun Wu <hu...@twitter.com>
AuthorDate: Wed May 2 14:04:06 2018 -0700
move downloader registry map to yaml
---
heron/config/src/yaml/conf/aurora/downloader.yaml | 6 +
.../config/src/yaml/conf/examples/downloader.yaml | 6 +
.../src/yaml/conf/kubernetes/downloader.yaml | 6 +
heron/config/src/yaml/conf/local/downloader.yaml | 6 +
heron/config/src/yaml/conf/localzk/downloader.yaml | 6 +
heron/config/src/yaml/conf/nomad/downloader.yaml | 6 +
heron/config/src/yaml/conf/sandbox/downloader.yaml | 6 +
.../src/yaml/conf/standalone/downloader.yaml | 6 +
heron/config/src/yaml/conf/yarn/downloader.yaml | 6 +
heron/downloaders/src/java/BUILD | 1 +
.../apache/heron/downloader/DownloadRunner.java | 163 ++++++++++++++++++++-
.../java/org/apache/heron/downloader/Registry.java | 27 +---
.../org/apache/heron/downloader/RegistryTest.java | 14 +-
.../org/apache/heron/spi/common/ConfigLoader.java | 2 +
.../java/org/apache/heron/spi/common/Context.java | 4 +
.../src/java/org/apache/heron/spi/common/Key.java | 23 +--
.../apache/heron/spi/common/ConfigLoaderTest.java | 5 +-
scripts/packages/BUILD | 8 +
tools/rules/heron_client.bzl | 6 +
19 files changed, 264 insertions(+), 43 deletions(-)
diff --git a/heron/config/src/yaml/conf/aurora/downloader.yaml b/heron/config/src/yaml/conf/aurora/downloader.yaml
new file mode 100644
index 0000000..203a106
--- /dev/null
+++ b/heron/config/src/yaml/conf/aurora/downloader.yaml
@@ -0,0 +1,6 @@
+# downloader class for protocols
+heron.downloader.registry:
+ http: org.apache.heron.downloader.HttpDownloader
+ https: org.apache.heron.downloader.HttpDownloader
+ distributedlog: org.apache.heron.downloader.DLDownloader
+ file: org.apache.heron.downloader.FileDownloader
diff --git a/heron/config/src/yaml/conf/examples/downloader.yaml b/heron/config/src/yaml/conf/examples/downloader.yaml
new file mode 100644
index 0000000..203a106
--- /dev/null
+++ b/heron/config/src/yaml/conf/examples/downloader.yaml
@@ -0,0 +1,6 @@
+# downloader class for protocols
+heron.downloader.registry:
+ http: org.apache.heron.downloader.HttpDownloader
+ https: org.apache.heron.downloader.HttpDownloader
+ distributedlog: org.apache.heron.downloader.DLDownloader
+ file: org.apache.heron.downloader.FileDownloader
diff --git a/heron/config/src/yaml/conf/kubernetes/downloader.yaml b/heron/config/src/yaml/conf/kubernetes/downloader.yaml
new file mode 100644
index 0000000..203a106
--- /dev/null
+++ b/heron/config/src/yaml/conf/kubernetes/downloader.yaml
@@ -0,0 +1,6 @@
+# downloader class for protocols
+heron.downloader.registry:
+ http: org.apache.heron.downloader.HttpDownloader
+ https: org.apache.heron.downloader.HttpDownloader
+ distributedlog: org.apache.heron.downloader.DLDownloader
+ file: org.apache.heron.downloader.FileDownloader
diff --git a/heron/config/src/yaml/conf/local/downloader.yaml b/heron/config/src/yaml/conf/local/downloader.yaml
new file mode 100644
index 0000000..203a106
--- /dev/null
+++ b/heron/config/src/yaml/conf/local/downloader.yaml
@@ -0,0 +1,6 @@
+# downloader class for protocols
+heron.downloader.registry:
+ http: org.apache.heron.downloader.HttpDownloader
+ https: org.apache.heron.downloader.HttpDownloader
+ distributedlog: org.apache.heron.downloader.DLDownloader
+ file: org.apache.heron.downloader.FileDownloader
diff --git a/heron/config/src/yaml/conf/localzk/downloader.yaml b/heron/config/src/yaml/conf/localzk/downloader.yaml
new file mode 100644
index 0000000..203a106
--- /dev/null
+++ b/heron/config/src/yaml/conf/localzk/downloader.yaml
@@ -0,0 +1,6 @@
+# downloader class for protocols
+heron.downloader.registry:
+ http: org.apache.heron.downloader.HttpDownloader
+ https: org.apache.heron.downloader.HttpDownloader
+ distributedlog: org.apache.heron.downloader.DLDownloader
+ file: org.apache.heron.downloader.FileDownloader
diff --git a/heron/config/src/yaml/conf/nomad/downloader.yaml b/heron/config/src/yaml/conf/nomad/downloader.yaml
new file mode 100644
index 0000000..203a106
--- /dev/null
+++ b/heron/config/src/yaml/conf/nomad/downloader.yaml
@@ -0,0 +1,6 @@
+# downloader class for protocols
+heron.downloader.registry:
+ http: org.apache.heron.downloader.HttpDownloader
+ https: org.apache.heron.downloader.HttpDownloader
+ distributedlog: org.apache.heron.downloader.DLDownloader
+ file: org.apache.heron.downloader.FileDownloader
diff --git a/heron/config/src/yaml/conf/sandbox/downloader.yaml b/heron/config/src/yaml/conf/sandbox/downloader.yaml
new file mode 100644
index 0000000..203a106
--- /dev/null
+++ b/heron/config/src/yaml/conf/sandbox/downloader.yaml
@@ -0,0 +1,6 @@
+# downloader class for protocols
+heron.downloader.registry:
+ http: org.apache.heron.downloader.HttpDownloader
+ https: org.apache.heron.downloader.HttpDownloader
+ distributedlog: org.apache.heron.downloader.DLDownloader
+ file: org.apache.heron.downloader.FileDownloader
diff --git a/heron/config/src/yaml/conf/standalone/downloader.yaml b/heron/config/src/yaml/conf/standalone/downloader.yaml
new file mode 100644
index 0000000..203a106
--- /dev/null
+++ b/heron/config/src/yaml/conf/standalone/downloader.yaml
@@ -0,0 +1,6 @@
+# downloader class for protocols
+heron.downloader.registry:
+ http: org.apache.heron.downloader.HttpDownloader
+ https: org.apache.heron.downloader.HttpDownloader
+ distributedlog: org.apache.heron.downloader.DLDownloader
+ file: org.apache.heron.downloader.FileDownloader
diff --git a/heron/config/src/yaml/conf/yarn/downloader.yaml b/heron/config/src/yaml/conf/yarn/downloader.yaml
new file mode 100644
index 0000000..203a106
--- /dev/null
+++ b/heron/config/src/yaml/conf/yarn/downloader.yaml
@@ -0,0 +1,6 @@
+# downloader class for protocols
+heron.downloader.registry:
+ http: org.apache.heron.downloader.HttpDownloader
+ https: org.apache.heron.downloader.HttpDownloader
+ distributedlog: org.apache.heron.downloader.DLDownloader
+ file: org.apache.heron.downloader.FileDownloader
diff --git a/heron/downloaders/src/java/BUILD b/heron/downloaders/src/java/BUILD
index d61f18a..e854b99 100644
--- a/heron/downloaders/src/java/BUILD
+++ b/heron/downloaders/src/java/BUILD
@@ -2,6 +2,7 @@ package(default_visibility = ["//visibility:public"])
downloader_deps = [
"@org_apache_distributedlog_core//jar",
+ "@commons_cli_commons_cli//jar",
"//heron/spi/src/java:common-spi-java",
"//heron/io/dlog/src/java:dlog-lib",
"//third_party/java:commons-compress",
diff --git a/heron/downloaders/src/java/org/apache/heron/downloader/DownloadRunner.java b/heron/downloaders/src/java/org/apache/heron/downloader/DownloadRunner.java
index e094f0d..92919f8 100644
--- a/heron/downloaders/src/java/org/apache/heron/downloader/DownloadRunner.java
+++ b/heron/downloaders/src/java/org/apache/heron/downloader/DownloadRunner.java
@@ -17,18 +17,164 @@ import java.io.File;
import java.net.URI;
import java.nio.file.Path;
import java.nio.file.Paths;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.CommandLineParser;
+import org.apache.commons.cli.DefaultParser;
+import org.apache.commons.cli.HelpFormatter;
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.Options;
+import org.apache.commons.cli.ParseException;
+import org.apache.heron.spi.common.Config;
+import org.apache.heron.spi.common.ConfigLoader;
+import org.apache.heron.spi.common.Key;
public final class DownloadRunner {
+ public enum DownloaderMode {
+ cluster,
+ local
+ }
+
+ private enum CliArgs {
+ HERON_HOME("heron_home"),
+ CONFIG_PATH("config_path"),
+ MODE("mode"),
+ TOPOLOGY_PACKAGE_URI("topology_package_uri"),
+ EXTRACT_DESTINATION("extract_destination");
+
+ private String text;
+
+ CliArgs(String name) {
+ this.text = name;
+ }
+ }
+
+ // Print usage options
+ private static void usage(Options options) {
+ HelpFormatter formatter = new HelpFormatter();
+ formatter.printHelp(DownloadRunner.class.getSimpleName(), options);
+ }
+
+ // construct command line help options
+ private static Options constructHelpOptions() {
+ Options options = new Options();
+ Option help = Option.builder("h")
+ .desc("List all options and their description")
+ .longOpt("help")
+ .build();
+
+ options.addOption(help);
+ return options;
+ }
+
+ // Construct all required command line options
+ private static Options constructCliOptions() {
+ Options options = new Options();
+
+ Option packageUri = Option.builder("u")
+ .desc("Uri indicating from where to download the file")
+ .longOpt(CliArgs.TOPOLOGY_PACKAGE_URI.text)
+ .hasArgs()
+ .argName(CliArgs.TOPOLOGY_PACKAGE_URI.text)
+ .required()
+ .build();
+
+ Option destination = Option.builder("f")
+ .desc("Destination to store the downloaded file")
+ .longOpt(CliArgs.EXTRACT_DESTINATION.text)
+ .hasArgs()
+ .argName(CliArgs.EXTRACT_DESTINATION.text)
+ .required()
+ .build();
+
+ Option heronHome = Option.builder("d")
+ .desc("Directory where heron is installed")
+ .longOpt(CliArgs.HERON_HOME.text)
+ .hasArgs()
+ .argName("heron home dir")
+ .build();
+
+ Option configFile = Option.builder("p")
+ .desc("Path of the config files")
+ .longOpt(CliArgs.CONFIG_PATH.text)
+ .hasArgs()
+ .argName("config path")
+ .build();
+
+ // candidates:
+ // local: download to client local machine
+ // cluster: download into the container in the cloud
+ Option mode = Option.builder("m")
+ .desc("download mode, cluster or local")
+ .longOpt(CliArgs.MODE.text)
+ .hasArg()
+ .argName("download mode")
+ .build();
+
+ options.addOption(packageUri);
+ options.addOption(destination);
+ options.addOption(heronHome);
+ options.addOption(configFile);
+ options.addOption(mode);
+
+ return options;
+ }
+
+
// takes topology package URI and extracts it to a directory
public static void main(String[] args) throws Exception {
- if (args.length != 2) {
- System.err.println("Usage: downloader <topology-package-uri> <extract-destination>");
+ CommandLineParser parser = new DefaultParser();
+ Options slaManagerCliOptions = constructCliOptions();
+
+ // parse the help options first.
+ Options helpOptions = constructHelpOptions();
+ CommandLine cmd = parser.parse(helpOptions, args, true);
+ if (cmd.hasOption("h")) {
+ usage(slaManagerCliOptions);
return;
}
- final String uri = args[0];
- final String destination = args[1];
+ try {
+ cmd = parser.parse(slaManagerCliOptions, args);
+ } catch (ParseException e) {
+ usage(slaManagerCliOptions);
+ throw new RuntimeException("Error parsing command line options: ", e);
+ }
+
+ DownloaderMode mode = DownloaderMode.cluster;
+ if (cmd.hasOption(CliArgs.MODE.text)) {
+ mode = DownloaderMode.valueOf(cmd.getOptionValue(CliArgs.MODE.text, null));
+ }
+
+ Config config;
+ switch (mode) {
+ case cluster:
+ config = Config.toClusterMode(Config.newBuilder()
+ .putAll(ConfigLoader.loadClusterConfig())
+ .build());
+ break;
+
+ case local:
+ if (!cmd.hasOption(CliArgs.HERON_HOME.text) || !cmd.hasOption(CliArgs.CONFIG_PATH.text)) {
+ throw new IllegalArgumentException("Missing heron_home or config_path argument");
+ }
+ String heronHome = cmd.getOptionValue(CliArgs.HERON_HOME.text, null);
+ String configPath = cmd.getOptionValue(CliArgs.CONFIG_PATH.text, null);
+ config = Config.toLocalMode(Config.newBuilder()
+ .putAll(ConfigLoader.loadConfig(heronHome, configPath, null, null))
+ .build());
+ break;
+
+ default:
+ throw new IllegalArgumentException(
+ "Invalid mode: " + cmd.getOptionValue(CliArgs.MODE.text));
+ }
+
+ final String uri = cmd.getOptionValue(CliArgs.TOPOLOGY_PACKAGE_URI.text, null);
+ final String destination = cmd.getOptionValue(CliArgs.EXTRACT_DESTINATION.text, null);
final URI topologyLocation = new URI(uri);
final Path topologyDestination = Paths.get(destination);
@@ -38,7 +184,14 @@ public final class DownloadRunner {
file.mkdirs();
}
- final Downloader downloader = Registry.get().getDownloader(topologyLocation);
+ Map<String, Class<? extends Downloader>> downloaders = new HashMap<>();
+ for (Map.Entry<String, Object> e
+ : ((Map<String, Object>) config.get(Key.DOWNLOADER_PROTOCOLS)).entrySet()) {
+ Class clazz = Class.forName((String) e.getValue());
+ downloaders.put(e.getKey(), clazz);
+ }
+
+ final Downloader downloader = Registry.getDownloader(downloaders, topologyLocation);
downloader.download(topologyLocation, topologyDestination);
}
diff --git a/heron/downloaders/src/java/org/apache/heron/downloader/Registry.java b/heron/downloaders/src/java/org/apache/heron/downloader/Registry.java
index 16e3677..1e7c910 100644
--- a/heron/downloaders/src/java/org/apache/heron/downloader/Registry.java
+++ b/heron/downloaders/src/java/org/apache/heron/downloader/Registry.java
@@ -15,39 +15,22 @@ package org.apache.heron.downloader;
import java.lang.reflect.InvocationTargetException;
import java.net.URI;
-import java.util.HashMap;
import java.util.Map;
final class Registry {
- private static final Map<String, Class<? extends Downloader>> DOWNLOADERS =
- new HashMap<>();
+ private Registry() { }
- static {
- DOWNLOADERS.put("http", HttpDownloader.class);
- DOWNLOADERS.put("https", HttpDownloader.class);
- DOWNLOADERS.put("distributedlog", DLDownloader.class);
- DOWNLOADERS.put("file", FileDownloader.class);
- }
-
- private static final Registry INSTANCE = new Registry();
-
- private Registry() {
- }
-
- static Registry get() {
- return INSTANCE;
- }
-
- Downloader getDownloader(URI uri) throws Exception {
+ public static Downloader getDownloader(
+ Map<String, Class<? extends Downloader>> downloaders, URI uri) throws Exception {
final String scheme = uri.getScheme().toLowerCase();
- if (!DOWNLOADERS.containsKey(scheme)) {
+ if (!downloaders.containsKey(scheme)) {
throw new RuntimeException(
String.format("Unable to create downloader unsupported uri %s", uri.toString()));
}
try {
- final Class<? extends Downloader> downloaderClass = DOWNLOADERS.get(scheme);
+ final Class<? extends Downloader> downloaderClass = downloaders.get(scheme);
return downloaderClass.getConstructor().newInstance();
} catch (InstantiationException | IllegalAccessException
diff --git a/heron/downloaders/tests/java/org/apache/heron/downloader/RegistryTest.java b/heron/downloaders/tests/java/org/apache/heron/downloader/RegistryTest.java
index 75fdc52..b5dc59c 100644
--- a/heron/downloaders/tests/java/org/apache/heron/downloader/RegistryTest.java
+++ b/heron/downloaders/tests/java/org/apache/heron/downloader/RegistryTest.java
@@ -14,6 +14,8 @@
package org.apache.heron.downloader;
import java.net.URI;
+import java.util.HashMap;
+import java.util.Map;
import org.junit.Test;
@@ -23,14 +25,20 @@ public class RegistryTest {
@Test
public void testGetDownloader() throws Exception {
+ Map<String, Class<? extends Downloader>> downloaders = new HashMap<>();
+ downloaders.put("http", HttpDownloader.class);
+ downloaders.put("https", HttpDownloader.class);
+ downloaders.put("distributedlog", DLDownloader.class);
+ downloaders.put("file", FileDownloader.class);
+
URI httpUri = URI.create("http://127.0.0.1/test/http");
- Downloader downloader = Registry.get().getDownloader(httpUri);
+ Downloader downloader = Registry.getDownloader(downloaders, httpUri);
assertTrue(downloader instanceof HttpDownloader);
URI httpsUri = URI.create("https://127.0.0.1/test/http");
- downloader = Registry.get().getDownloader(httpsUri);
+ downloader = Registry.getDownloader(downloaders, httpsUri);
assertTrue(downloader instanceof HttpDownloader);
URI dlUri = URI.create("distributedlog://127.0.0.1/test/distributedlog");
- downloader = Registry.get().getDownloader(dlUri);
+ downloader = Registry.getDownloader(downloaders, dlUri);
assertTrue(downloader instanceof DLDownloader);
}
diff --git a/heron/spi/src/java/org/apache/heron/spi/common/ConfigLoader.java b/heron/spi/src/java/org/apache/heron/spi/common/ConfigLoader.java
index 886f8d0..1ef1967 100644
--- a/heron/spi/src/java/org/apache/heron/spi/common/ConfigLoader.java
+++ b/heron/spi/src/java/org/apache/heron/spi/common/ConfigLoader.java
@@ -62,6 +62,7 @@ public final class ConfigLoader {
.putAll(loadConfig(Context.schedulerFile(localConfig)))
.putAll(loadConfig(Context.stateManagerFile(localConfig)))
.putAll(loadConfig(Context.uploaderFile(localConfig)))
+ .putAll(loadConfig(Context.downloaderFile(localConfig)))
.putAll(loadConfig(Context.statefulConfigFile(localConfig)))
.putAll(loadConfig(releaseFile))
.putAll(loadConfig(overrideConfigFile));
@@ -85,6 +86,7 @@ public final class ConfigLoader {
.putAll(loadConfig(Context.schedulerFile(clusterConfig)))
.putAll(loadConfig(Context.stateManagerFile(clusterConfig)))
.putAll(loadConfig(Context.uploaderFile(clusterConfig)))
+ .putAll(loadConfig(Context.downloaderFile(clusterConfig)))
.putAll(loadConfig(Context.statefulConfigFile(clusterConfig)));
// Add the override config at the end to replace any existing configs
diff --git a/heron/spi/src/java/org/apache/heron/spi/common/Context.java b/heron/spi/src/java/org/apache/heron/spi/common/Context.java
index b74ffec..381381a 100644
--- a/heron/spi/src/java/org/apache/heron/spi/common/Context.java
+++ b/heron/spi/src/java/org/apache/heron/spi/common/Context.java
@@ -149,6 +149,10 @@ public class Context {
return cfg.getStringValue(Key.UPLOADER_YAML);
}
+ public static String downloaderFile(Config cfg) {
+ return cfg.getStringValue(Key.DOWNLOADER_YAML);
+ }
+
public static String schedulerJar(Config cfg) {
return cfg.getStringValue(Key.SCHEDULER_JAR);
}
diff --git a/heron/spi/src/java/org/apache/heron/spi/common/Key.java b/heron/spi/src/java/org/apache/heron/spi/common/Key.java
index 7aa7320..1af5f2c 100644
--- a/heron/spi/src/java/org/apache/heron/spi/common/Key.java
+++ b/heron/spi/src/java/org/apache/heron/spi/common/Key.java
@@ -36,16 +36,17 @@ public enum Key {
JAVA_HOME ("heron.directory.java.home", "${JAVA_HOME}"),
//keys for heron configuration files
- CLUSTER_YAML ("heron.config.file.cluster.yaml", "${HERON_CONF}/cluster.yaml"),
- CLIENT_YAML ("heron.config.file.client.yaml", "${HERON_CONF}/client.yaml"),
- HEALTHMGR_YAML ("heron.config.file.healthmgr.yaml", "${HERON_CONF}/healthmgr.yaml"),
- METRICS_YAML ("heron.config.file.metrics.yaml", "${HERON_CONF}/metrics_sinks.yaml"),
- PACKING_YAML ("heron.config.file.packing.yaml", "${HERON_CONF}/packing.yaml"),
- SCHEDULER_YAML ("heron.config.file.scheduler.yaml", "${HERON_CONF}/scheduler.yaml"),
- STATEMGR_YAML ("heron.config.file.statemgr.yaml", "${HERON_CONF}/statemgr.yaml"),
- SYSTEM_YAML ("heron.config.file.system.yaml", "${HERON_CONF}/heron_internals.yaml"),
- UPLOADER_YAML ("heron.config.file.uploader.yaml", "${HERON_CONF}/uploader.yaml"),
- STATEFUL_YAML ("heron.config.file.stateful.yaml", "${HERON_CONF}/stateful.yaml"),
+ CLUSTER_YAML ("heron.config.file.cluster.yaml", "${HERON_CONF}/cluster.yaml"),
+ CLIENT_YAML ("heron.config.file.client.yaml", "${HERON_CONF}/client.yaml"),
+ HEALTHMGR_YAML ("heron.config.file.healthmgr.yaml", "${HERON_CONF}/healthmgr.yaml"),
+ METRICS_YAML ("heron.config.file.metrics.yaml", "${HERON_CONF}/metrics_sinks.yaml"),
+ PACKING_YAML ("heron.config.file.packing.yaml", "${HERON_CONF}/packing.yaml"),
+ SCHEDULER_YAML ("heron.config.file.scheduler.yaml", "${HERON_CONF}/scheduler.yaml"),
+ STATEMGR_YAML ("heron.config.file.statemgr.yaml", "${HERON_CONF}/statemgr.yaml"),
+ SYSTEM_YAML ("heron.config.file.system.yaml", "${HERON_CONF}/heron_internals.yaml"),
+ UPLOADER_YAML ("heron.config.file.uploader.yaml", "${HERON_CONF}/uploader.yaml"),
+ DOWNLOADER_YAML ("heron.config.file.downloader.yaml", "${HERON_CONF}/downloader.yaml"),
+ STATEFUL_YAML ("heron.config.file.stateful.yaml", "${HERON_CONF}/stateful.yaml"),
//keys for config provided in the command line
CLUSTER ("heron.config.cluster", Type.STRING),
@@ -148,6 +149,7 @@ public enum Key {
COMPONENT_JVM_OPTS_IN_BASE64 ("heron.runtime.component.jvm.opts.in.base64", Type.STRING),
INSTANCE_JVM_OPTS_IN_BASE64 ("heron.runtime.instance.jvm.opts.in.base64", Type.STRING),
NUM_CONTAINERS ("heron.runtime.num.containers", Type.INTEGER),
+ DOWNLOADER_PROTOCOLS ("heron.downloader.registry", Type.STRING),
//release info
HERON_RELEASE_PACKAGE ("heron.release.package", Type.STRING),
@@ -184,7 +186,6 @@ public enum Key {
// To enable it, change the config from "disabled" to "prompt".
UPDATE_PROMPT ("heron.command.update.prompt", "disabled");
-
private final String value;
private final Object defaultValue;
private final Type type;
diff --git a/heron/spi/tests/java/org/apache/heron/spi/common/ConfigLoaderTest.java b/heron/spi/tests/java/org/apache/heron/spi/common/ConfigLoaderTest.java
index 94352e8..9343f11 100644
--- a/heron/spi/tests/java/org/apache/heron/spi/common/ConfigLoaderTest.java
+++ b/heron/spi/tests/java/org/apache/heron/spi/common/ConfigLoaderTest.java
@@ -82,7 +82,7 @@ public class ConfigLoaderTest {
String heronConfigPath) {
// assert that the config filenames passed to loadConfig are never null. If they are, the
// configs defaults are not producing the config files.
- PowerMockito.verifyStatic(times(10));
+ PowerMockito.verifyStatic(times(11));
ConfigLoader.loadConfig(isNotNull(String.class));
PowerMockito.verifyStatic(never());
ConfigLoader.loadConfig(isNull(String.class));
@@ -90,7 +90,7 @@ public class ConfigLoaderTest {
// addFromFile with an empty map means that the config file was not found. Of the 9 files that
// are attempted to be loaded, all but 3 should be found (clientConfig, overrideConfigFile and
// releaseFile do not exist)
- PowerMockito.verifyStatic(times(3));
+ PowerMockito.verifyStatic(times(4));
ConfigLoader.addFromFile(eq(new HashMap<String, Object>()));
Set<String> tokenizedValues = new TreeSet<>();
@@ -119,6 +119,7 @@ public class ConfigLoaderTest {
assertKeyValue(config, Key.STATEMGR_YAML, heronConfigPath + "/statemgr.yaml");
assertKeyValue(config, Key.SYSTEM_YAML, heronConfigPath + "/heron_internals.yaml");
assertKeyValue(config, Key.UPLOADER_YAML, heronConfigPath + "/uploader.yaml");
+ assertKeyValue(config, Key.DOWNLOADER_YAML, heronConfigPath + "/downloader.yaml");
String binPath = config.getStringValue(Key.HERON_BIN);
assertKeyValue(config, Key.EXECUTOR_BINARY, binPath + "/heron-executor");
diff --git a/scripts/packages/BUILD b/scripts/packages/BUILD
index 6076234..6425b36 100644
--- a/scripts/packages/BUILD
+++ b/scripts/packages/BUILD
@@ -10,6 +10,7 @@ load("//tools/rules:heron_client.bzl", "heron_client_lib_metricscachemgr_files")
load("//tools/rules:heron_client.bzl", "heron_client_lib_packing_files")
load("//tools/rules:heron_client.bzl", "heron_client_lib_statemgr_files")
load("//tools/rules:heron_client.bzl", "heron_client_lib_uploader_files")
+load("//tools/rules:heron_client.bzl", "heron_client_lib_downloader_files")
load("//tools/rules:heron_client.bzl", "heron_client_lib_healthmgr_files")
load("//tools/rules:heron_core.bzl", "heron_core_files")
@@ -266,6 +267,12 @@ pkg_tar(
)
pkg_tar(
+ name = "heron-lib-downloader",
+ package_dir = "lib/downloader",
+ srcs = heron_client_lib_downloader_files(),
+)
+
+pkg_tar(
name = "heron-lib-healthmgr",
package_dir = "lib/healthmgr",
srcs = heron_client_lib_healthmgr_files(),
@@ -328,6 +335,7 @@ pkg_tar(
":heron-lib-statemgr",
":heron-lib-metricscachemgr",
":heron-lib-uploader",
+ ":heron-lib-downloader",
],
)
diff --git a/tools/rules/heron_client.bzl b/tools/rules/heron_client.bzl
index b62b90a..48c7d38 100644
--- a/tools/rules/heron_client.bzl
+++ b/tools/rules/heron_client.bzl
@@ -7,6 +7,7 @@ def heron_client_bin_files():
"//heron/tools/explorer/src/python:heron-explorer",
"//heron/tools/admin/src/python:heron-admin",
"//third_party/nomad:heron-nomad",
+ "//heron/downloaders/src/shell:heron-downloader"
]
def heron_client_conf_files():
@@ -85,3 +86,8 @@ def heron_client_lib_third_party_files():
"@org_slf4j_slf4j_api//jar",
"@org_slf4j_slf4j_jdk14//jar",
]
+
+def heron_client_lib_downloader_files():
+ return [
+ "//heron/downloaders/src/java:heron-downloader",
+ ]
--
To stop receiving notification emails like this one, please contact
huijun@apache.org.