You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@heron.apache.org by sa...@apache.org on 2022/05/09 21:35:59 UTC
[incubator-heron] 04/07: [Config] Removed Aurora from configs.
This is an automated email from the ASF dual-hosted git repository.
saadurrahman pushed a commit to branch saadurrahman/3829-Deprecate-Apache-Aurora-dev
in repository https://gitbox.apache.org/repos/asf/incubator-heron.git
commit 825e818d92376de5ada58869ffd0c12771474ed6
Author: Saad Ur Rahman <sa...@apache.org>
AuthorDate: Mon May 9 17:21:59 2022 -0400
[Config] Removed Aurora from configs.
---
heron/config/src/yaml/BUILD | 15 +-
heron/config/src/yaml/conf/aurora/README | 1 -
heron/config/src/yaml/conf/aurora/client.yaml | 23 --
heron/config/src/yaml/conf/aurora/downloader.yaml | 23 --
heron/config/src/yaml/conf/aurora/heron.aurora | 93 -------
.../src/yaml/conf/aurora/heron_internals.yaml | 296 ---------------------
.../config/src/yaml/conf/aurora/metrics_sinks.yaml | 257 ------------------
heron/config/src/yaml/conf/aurora/packing.yaml | 20 --
heron/config/src/yaml/conf/aurora/scheduler.yaml | 61 -----
heron/config/src/yaml/conf/aurora/stateful.yaml | 38 ---
heron/config/src/yaml/conf/aurora/statemgr.yaml | 49 ----
heron/config/src/yaml/conf/aurora/uploader.yaml | 22 --
heron/config/src/yaml/conf/examples/README.md | 3 -
.../src/yaml/conf/examples/aurora_scheduler.yaml | 29 --
14 files changed, 3 insertions(+), 927 deletions(-)
diff --git a/heron/config/src/yaml/BUILD b/heron/config/src/yaml/BUILD
index aba545eccdd..2cdcf392e35 100644
--- a/heron/config/src/yaml/BUILD
+++ b/heron/config/src/yaml/BUILD
@@ -15,13 +15,9 @@ filegroup(
filegroup(
name = "conf-yaml",
srcs = glob(
- ["conf/**/*.aurora"] + [
- "conf/**/*.hcl",
- ] + [
- "conf/**/*.sh",
- ] + [
- "conf/**/*.yaml",
- ],
+ ["conf/**/*.hcl"] +
+ ["conf/**/*.sh"] +
+ ["conf/**/*.yaml"],
),
)
@@ -35,11 +31,6 @@ filegroup(
srcs = glob(["conf/sandbox/*.yaml"]),
)
-filegroup(
- name = "conf-aurora-yaml",
- srcs = glob(["conf/aurora/*"]),
-)
-
filegroup(
name = "conf-slurm-yaml",
srcs = glob(["conf/slurm/*"]),
diff --git a/heron/config/src/yaml/conf/aurora/README b/heron/config/src/yaml/conf/aurora/README
deleted file mode 100644
index 4a92300fe61..00000000000
--- a/heron/config/src/yaml/conf/aurora/README
+++ /dev/null
@@ -1 +0,0 @@
-Config customization is required before using.
\ No newline at end of file
diff --git a/heron/config/src/yaml/conf/aurora/client.yaml b/heron/config/src/yaml/conf/aurora/client.yaml
deleted file mode 100644
index f14d8fdd968..00000000000
--- a/heron/config/src/yaml/conf/aurora/client.yaml
+++ /dev/null
@@ -1,23 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-# location of the core package
-heron.package.core.uri: "file:///vagrant/.herondata/dist/heron-core-release.tar.gz"
-
-# Whether role/env is required to submit a topology. Default value is False.
-heron.config.is.role.required: True
-heron.config.is.env.required: True
diff --git a/heron/config/src/yaml/conf/aurora/downloader.yaml b/heron/config/src/yaml/conf/aurora/downloader.yaml
deleted file mode 100644
index 68805525670..00000000000
--- a/heron/config/src/yaml/conf/aurora/downloader.yaml
+++ /dev/null
@@ -1,23 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-# downloader class for protocols
-heron.downloader.registry:
- http: org.apache.heron.downloader.HttpDownloader
- https: org.apache.heron.downloader.HttpDownloader
- distributedlog: org.apache.heron.downloader.DLDownloader
- file: org.apache.heron.downloader.FileDownloader
diff --git a/heron/config/src/yaml/conf/aurora/heron.aurora b/heron/config/src/yaml/conf/aurora/heron.aurora
deleted file mode 100644
index 1826e7faa61..00000000000
--- a/heron/config/src/yaml/conf/aurora/heron.aurora
+++ /dev/null
@@ -1,93 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-"""
-Launch the topology as a single aurora job with multiple instances.
-The heron-executor is responsible for starting a tmanager (container 0)
-and regular stmgr/metricsmgr/instances (container index > 0).
-"""
-
-heron_core_release_uri = '{{CORE_PACKAGE_URI}}'
-heron_topology_jar_uri = '{{TOPOLOGY_PACKAGE_URI}}'
-core_release_file = "heron-core.tar.gz"
-topology_package_file = "topology.tar.gz"
-
-# --- processes ---
-fetch_heron_system = Process(
- name = 'fetch_heron_system',
- cmdline = 'curl %s -o %s && tar zxf %s' % (heron_core_release_uri, core_release_file, core_release_file)
-)
-
-fetch_user_package = Process(
- name = 'fetch_user_package',
- cmdline = 'curl %s -o %s && tar zxf %s' % (heron_topology_jar_uri, topology_package_file, topology_package_file)
-)
-
-command_to_start_executor = \
- '{{EXECUTOR_BINARY}}' \
- ' --shard={{mesos.instance}}' \
- ' --server-port={{thermos.ports[port1]}}' \
- ' --tmanager-controller-port={{thermos.ports[port2]}}' \
- ' --tmanager-stats-port={{thermos.ports[port3]}}' \
- ' --shell-port={{thermos.ports[http]}}' \
- ' --metrics-manager-port={{thermos.ports[port4]}}' \
- ' --scheduler-port={{thermos.ports[scheduler]}}' \
- ' --metricscache-manager-server-port={{thermos.ports[metricscachemgr_serverport]}}' \
- ' --metricscache-manager-stats-port={{thermos.ports[metricscachemgr_statsport]}}' \
- ' --checkpoint-manager-port={{thermos.ports[ckptmgr_port]}}' \
- ' {{TOPOLOGY_ARGUMENTS}}'
-
-launch_heron_executor = Process(
- name = 'launch_heron_executor',
- cmdline = command_to_start_executor,
- max_failures = 1
-)
-
-discover_profiler_port = Process(
- name = 'discover_profiler_port',
- cmdline = 'echo {{thermos.ports[yourkit]}} > yourkit.port'
-)
-
-# --- tasks ---
-heron_task = SequentialTask(
- name = 'setup_and_run',
- processes = [fetch_heron_system, fetch_user_package, launch_heron_executor, discover_profiler_port],
- resources = Resources(cpu = '{{CPUS_PER_CONTAINER}}', ram = '{{RAM_PER_CONTAINER}}', disk = '{{DISK_PER_CONTAINER}}')
-)
-
-# -- jobs ---
-jobs = [
- Job(
- name = '{{TOPOLOGY_NAME}}',
- cluster = '{{CLUSTER}}',
- role = '{{ROLE}}',
- environment = '{{ENVIRON}}',
- service = True,
- task = heron_task,
- instances = '{{NUM_CONTAINERS}}',
- announce = Announcer(primary_port = 'http', portmap = {'admin': 'http', 'health': 'http'}),
- health_check_config = HealthCheckConfig(
- health_checker = HealthCheckerConfig(
- http = HttpHealthChecker(endpoint='/health', expected_response='ok', expected_response_code=200)
- ),
- initial_interval_secs=30,
- interval_secs=10,
- max_consecutive_failures=2,
- timeout_secs=5
- )
- )
-]
diff --git a/heron/config/src/yaml/conf/aurora/heron_internals.yaml b/heron/config/src/yaml/conf/aurora/heron_internals.yaml
deleted file mode 100644
index 22685e4e1a1..00000000000
--- a/heron/config/src/yaml/conf/aurora/heron_internals.yaml
+++ /dev/null
@@ -1,296 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-################################################################################
-# Default values for various configs used inside Heron.
-################################################################################
-# All the config associated with time is in the unit of milli-seconds,
-# unless otherwise specified.
-################################################################################
-# All the config associated with data size is in the unit of bytes, unless
-# otherwise specified.
-################################################################################
-
-################################################################################
-# System level configs
-################################################################################
-
-### heron.* configs are general configurations over all componenets
-
-# The relative path to the logging directory
-heron.logging.directory: "log-files"
-
-# The maximum log file size in MB
-heron.logging.maximum.size.mb: 100
-
-# The maximum number of log files
-heron.logging.maximum.files: 5
-
-# The interval in seconds after which to check if the tmanager location has been fetched or not
-heron.check.tmanager.location.interval.sec: 120
-
-# The interval in seconds to prune logging files in C++
-heron.logging.prune.interval.sec: 300
-
-# The interval in seconds to flush log files in C++
-heron.logging.flush.interval.sec: 10
-
-# The threshold level to log error
-heron.logging.err.threshold: 3
-
-# The interval in seconds for different components to export metrics to metrics manager
-heron.metrics.export.interval.sec: 60
-
-# The maximum count of exceptions in one MetricPublisherPublishMessage protobuf
-heron.metrics.max.exceptions.per.message.count: 1024
-
-################################################################################
-# Configs related to Stream Manager, starts with heron.streammgr.*
-################################################################################
-
-# The tuple cache (used for batching) can be drained in two ways:
-# (a) Time based
-# (b) size based
-
-# The frequency in ms to drain the tuple cache in stream manager
-heron.streammgr.cache.drain.frequency.ms: 10
-
-# The sized based threshold in MB for buffering data tuples
-# waiting for checkpoint markers before giving up
-heron.streammgr.stateful.buffer.size.mb: 100
-
-# The sized based threshold in MB for draining the tuple cache
-heron.streammgr.cache.drain.size.mb: 100
-
-# For efficient acknowledgements
-heron.streammgr.xormgr.rotatingmap.nbuckets: 3
-
-# The max number of messages in the memory pool for each message type
-heron.streammgr.mempool.max.message.number: 512
-
-# The reconnect interval to other stream managers in secs for stream manager client
-heron.streammgr.client.reconnect.interval.sec: 1
-
-# The reconnect interval to tamster in second for stream manager client
-heron.streammgr.client.reconnect.tmanager.interval.sec: 10
-
-# The max reconnect attempts to tmanager for stream manager client
-heron.streammgr.client.reconnect.tmanager.max.attempts: 30
-
-# The maximum packet size in MB of stream manager's network options
-heron.streammgr.network.options.maximum.packet.mb: 10
-
-# The interval in seconds to send heartbeat
-heron.streammgr.tmanager.heartbeat.interval.sec: 10
-
-# Maximum batch size in MB to read by stream manager from socket
-heron.streammgr.connection.read.batch.size.mb: 1
-
-# Maximum batch size in MB to write by stream manager to socket
-heron.streammgr.connection.write.batch.size.mb: 1
-
-# Number of times we should wait to see a buffer full while enqueueing data
-# before declaring start of back pressure
-heron.streammgr.network.backpressure.threshold: 3
-
-# High water mark on the num in MB that can be left outstanding on a connection
-heron.streammgr.network.backpressure.highwatermark.mb: 100
-
-# Low water mark on the num in MB that can be left outstanding on a connection
-heron.streammgr.network.backpressure.lowwatermark.mb: 50
-
-################################################################################
-# Configs related to Topology Manager, starts with heron.tmanager.*
-################################################################################
-
-# The maximum interval in minutes of metrics to be kept in tmanager
-heron.tmanager.metrics.collector.maximum.interval.min: 180
-
-# The maximum time to retry to establish the tmanager
-heron.tmanager.establish.retry.times: 30
-
-# The interval to retry to establish the tmanager
-heron.tmanager.establish.retry.interval.sec: 1
-
-# Maximum packet size in MB of tmanager's network options to connect to stream managers
-heron.tmanager.network.server.options.maximum.packet.mb: 16
-
-# Maximum packet size in MB of tmanager's network options to connect to scheduler
-heron.tmanager.network.controller.options.maximum.packet.mb: 1
-
-# Maximum packet size in MB of tmanager's network options for stat queries
-heron.tmanager.network.stats.options.maximum.packet.mb: 1
-
-# The interval for tmanager to purge metrics from socket
-heron.tmanager.metrics.collector.purge.interval.sec: 60
-
-# The maximum # of exceptions to be stored in tmetrics collector, to prevent potential OOM
-heron.tmanager.metrics.collector.maximum.exception: 256
-
-# Should the metrics reporter bind on all interfaces
-heron.tmanager.metrics.network.bindallinterfaces: False
-
-# The timeout in seconds for stream mgr, compared with (current time - last heartbeat time)
-heron.tmanager.stmgr.state.timeout.sec: 60
-
-################################################################################
-# Configs related to Metrics Manager, starts with heron.metricsmgr.*
-################################################################################
-
-# The size of packets to read from socket will be determined by the minimal of:
-# (a) time based
-# (b) size based
-
-# Time based, the maximum batch time in ms for metricsmgr to read from socket
-heron.metricsmgr.network.read.batch.time.ms: 16
-
-# Size based, the maximum batch size in bytes to read from socket
-heron.metricsmgr.network.read.batch.size.bytes: 32768
-
-# The size of packets to write to socket will be determined by the minimum of
-# (a) time based
-# (b) size based
-
-# Time based, the maximum batch time in ms for metricsmgr to write to socket
-heron.metricsmgr.network.write.batch.time.ms: 16
-
-# Size based, the maximum batch size in bytes to write to socket
-heron.metricsmgr.network.write.batch.size.bytes: 32768
-
-# The maximum socket's send buffer size in bytes
-heron.metricsmgr.network.options.socket.send.buffer.size.bytes: 6553600
-
-# The maximum socket's received buffer size in bytes of metricsmgr's network options
-heron.metricsmgr.network.options.socket.received.buffer.size.bytes: 8738000
-
-# The maximum packet size that metrics mgr can read
-heron.metricsmgr.network.options.maximum.packetsize.bytes: 1048576
-
-################################################################################
-# Configs related to Heron Instance, starts with heron.instance.*
-################################################################################
-
-# The maximum size of packets that instance can read
-heron.instance.network.options.maximum.packetsize.bytes: 10485760
-
-# The queue capacity (num of items) in bolt for buffer packets to read from stream manager
-heron.instance.internal.bolt.read.queue.capacity: 128
-
-# The queue capacity (num of items) in bolt for buffer packets to write to stream manager
-heron.instance.internal.bolt.write.queue.capacity: 128
-
-# The queue capacity (num of items) in spout for buffer packets to read from stream manager
-heron.instance.internal.spout.read.queue.capacity: 1024
-
-# The queue capacity (num of items) in spout for buffer packets to write to stream manager
-heron.instance.internal.spout.write.queue.capacity: 128
-
-# The queue capacity (num of items) for metrics packets to write to metrics manager
-heron.instance.internal.metrics.write.queue.capacity: 128
-
-# The size of packets read from stream manager will be determined by the minimal of
-# (a) time based
-# (b) size based
-
-# Time based, the maximum batch time in ms for instance to read from stream manager per attempt
-heron.instance.network.read.batch.time.ms: 16
-
-# Size based, the maximum batch size in bytes to read from stream manager
-heron.instance.network.read.batch.size.bytes: 32768
-
-# The size of packets written to stream manager will be determined by the minimum of
-# (a) time based
-# (b) size based
-
-# Time based, the maximum batch time in ms for instance to write to stream manager per attempt
-heron.instance.network.write.batch.time.ms: 16
-
-# Size based, the maximum batch size in bytes to write to stream manager
-heron.instance.network.write.batch.size.bytes: 32768
-
-# The maximum socket's send buffer size in bytes
-heron.instance.network.options.socket.send.buffer.size.bytes: 6553600
-
-# The maximum socket's received buffer size in bytes of instance's network options
-heron.instance.network.options.socket.received.buffer.size.bytes: 8738000
-
-# The maximum # of data tuple to batch in a HeronDataTupleSet protobuf
-heron.instance.set.data.tuple.capacity: 1024
-
-# The maximum size in bytes of data tuple to batch in a HeronDataTupleSet protobuf
-heron.instance.set.data.tuple.size.bytes: 8388608
-
-# The maximum # of control tuple to batch in a HeronControlTupleSet protobuf
-heron.instance.set.control.tuple.capacity: 1024
-
-# The maximum time in ms for a spout to do acknowledgement per attempt, the ack batch could
-# also break if there are no more ack tuples to process
-heron.instance.ack.batch.time.ms: 128
-
-# The maximum time in ms for an spout instance to emit tuples per attempt
-heron.instance.emit.batch.time.ms: 16
-
-# The maximum batch size in bytes for an spout to emit tuples per attempt
-heron.instance.emit.batch.size.bytes: 32768
-
-# The maximum time in ms for an bolt instance to execute tuples per attempt
-heron.instance.execute.batch.time.ms: 16
-
-# The maximum batch size in bytes for an bolt instance to execute tuples per attempt
-heron.instance.execute.batch.size.bytes: 32768
-
-# The time to wait before the instance exits forcibly when uncaught exception happens
-heron.instance.force.exit.timeout.ms: 2000
-
-# Interval in seconds to reconnect to the stream manager, including the request timeout in connecting
-heron.instance.reconnect.streammgr.interval.sec: 5
-heron.instance.reconnect.streammgr.times: 60
-
-# Interval in seconds to reconnect to the metrics manager, including the request timeout in connecting
-heron.instance.reconnect.metricsmgr.interval.sec: 5
-heron.instance.reconnect.metricsmgr.times: 60
-
-# The interval in second for an instance to sample its system metrics, for instance, CPU load.
-heron.instance.metrics.system.sample.interval.sec: 10
-
-# For efficient acknowledgement
-heron.instance.acknowledgement.nbuckets: 10
-
-################################################################################
-# For dynamically tuning the available sizes in the interval read & write queues
-# to provide high performance while avoiding GC issues
-################################################################################
-
-# The expected size on read queue in bolt
-heron.instance.tuning.expected.bolt.read.queue.size: 8
-
-# The expected size on write queue in bolt
-heron.instance.tuning.expected.bolt.write.queue.size: 8
-
-# The expected size on read queue in spout
-heron.instance.tuning.expected.spout.read.queue.size: 512
-
-# The exepected size on write queue in spout
-heron.instance.tuning.expected.spout.write.queue.size: 8
-
-# The expected size on metrics write queue
-heron.instance.tuning.expected.metrics.write.queue.size: 8
-
-heron.instance.tuning.current.sample.weight: 0.8
-
-# Interval in ms to tune the size of in & out data queue in instance
-heron.instance.tuning.interval.ms: 100
diff --git a/heron/config/src/yaml/conf/aurora/metrics_sinks.yaml b/heron/config/src/yaml/conf/aurora/metrics_sinks.yaml
deleted file mode 100644
index 6cac2f13c9e..00000000000
--- a/heron/config/src/yaml/conf/aurora/metrics_sinks.yaml
+++ /dev/null
@@ -1,257 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-########### These all have default values as shown
-
-# We would specify the unique sink-id first
-sinks:
- - file-sink
- - tmanager-sink
- - metricscache-sink
-
-########### Now we would specify the detailed configuration for every unique sink
-########### Syntax: sink-id: - option(s)
-
-########### option class is required as we need to instantiate a new instance by reflection
-########### option flush-frequency-ms is required to invoke flush() at interval
-########### option sink-restart-attempts, representsing # of times to restart a sink when it throws exceptions and dies.
-########### If this option is missed, default value 0 would be supplied; negative value represents to restart it forever.
-
-########### Other options would be constructed as an immutable map passed to IMetricsSink's init(Map conf) as argument,
-########### We would be able to fetch value by conf.get(options), for instance:
-########### We could get "org.apache.heron.metricsmgr.sink.FileSink" if conf.get("class") is called inside file-sink's instance
-
-### Config for file-sink
-file-sink:
- class: "org.apache.heron.metricsmgr.sink.FileSink"
- flush-frequency-ms: 60000 # 1 min
- sink-restart-attempts: -1 # Forever
- filename-output: "metrics.json" # File for metrics to write to
- file-maximum: 5 # maximum number of file saved in disk
-
-### Config for tmanager-sink
-tmanager-sink:
- class: "org.apache.heron.metricsmgr.sink.tmanager.TManagerSink"
- flush-frequency-ms: 60000
- sink-restart-attempts: -1 # Forever
- tmanager-location-check-interval-sec: 5
- tmanager-client:
- reconnect-interval-second: 5 # The re-connect interval to TManager from TManagerClient
- # The size of packets written to TManager will be determined by the minimal of: (a) time based (b) size based
- network-write-batch-size-bytes: 32768 # Size based, the maximum batch size in bytes to write to TManager
- network-write-batch-time-ms: 16 # Time based, the maximum batch time in ms for Metrics Manager to write to TManager per attempt
- network-read-batch-size-bytes: 32768 # Size based, the maximum batch size in bytes to write to TManager
- network-read-batch-time-ms: 16 # Time based, the maximum batch time in ms for Metrics Manager to write to TManager per attempt
- socket-send-buffer-size-bytes: 6553600 # The maximum socket's send buffer size in bytes
- socket-received-buffer-size-bytes: 8738000 # The maximum socket's received buffer size in bytes
- tmanager-metrics-type:
- "__emit-count": SUM
- "__execute-count": SUM
- "__fail-count": SUM
- "__ack-count": SUM
- "__complete-latency": AVG
- "__execute-latency": AVG
- "__process-latency": AVG
- "__jvm-uptime-secs": LAST
- "__jvm-process-cpu-load": LAST
- "__jvm-memory-used-mb": LAST
- "__jvm-memory-mb-total": LAST
- "__jvm-gc-collection-time-ms": LAST
- "__server/__time_spent_back_pressure_initiated": SUM
- "__time_spent_back_pressure_by_compid": SUM
-
-### Config for metricscache-sink
-metricscache-sink:
- class: "org.apache.heron.metricsmgr.sink.metricscache.MetricsCacheSink"
- flush-frequency-ms: 60000
- sink-restart-attempts: -1 # Forever
- metricscache-location-check-interval-sec: 5
- metricscache-client:
- reconnect-interval-second: 5 # The re-connect interval to TManager from TManagerClient
- # The size of packets written to TManager will be determined by the minimal of: (a) time based (b) size based
- network-write-batch-size-bytes: 32768 # Size based, the maximum batch size in bytes to write to TManager
- network-write-batch-time-ms: 16 # Time based, the maximum batch time in ms for Metrics Manager to write to TManager per attempt
- network-read-batch-size-bytes: 32768 # Size based, the maximum batch size in bytes to write to TManager
- network-read-batch-time-ms: 16 # Time based, the maximum batch time in ms for Metrics Manager to write to TManager per attempt
- socket-send-buffer-size-bytes: 6553600 # The maximum socket's send buffer size in bytes
- socket-received-buffer-size-bytes: 8738000 # The maximum socket's received buffer size in bytes
- metricscache-metrics-type:
- "__emit-count": SUM
- "__execute-count": SUM
- "__fail-count": SUM
- "__ack-count": SUM
- "__complete-latency": AVG
- "__execute-latency": AVG
- "__process-latency": AVG
- "__jvm-uptime-secs": LAST
- "__jvm-process-cpu-load": LAST
- "__jvm-memory-used-mb": LAST
- "__jvm-memory-mb-total": LAST
- "__jvm-gc-collection-time-ms": LAST
- "__server/__time_spent_back_pressure_initiated": SUM
- "__time_spent_back_pressure_by_compid": SUM
-
-### Config for prometheus-sink
-# prometheus-sink:
-# class: "org.apache.heron.metricsmgr.sink.PrometheusSink"
-# port: 8080 # The port on which to run (either port or port-file are mandatory)
-# path: /metrics # The path on which to publish the metrics (mandatory)
-# flat-metrics: true # By default the web-sink will publish a flat "name -> value" json map
-# include-topology-name: true # Include topology name in metric name (default false)
-# metrics-cache-max-size: 1000000 # Max number of metrics cached and published (default 1000000)
-# metrics-cache-ttl-sec: 600 # Time in seconds after which a metric that was collected will stopped being published (default 600)
-# rules:
-# # __jvm-peak-usage/G1-Survivor-Space-committed": "9",
-# - pattern: __jvm-(.+)/(.+)
-# name: jvm_$1_$2
-# attrNameSnakeCase: true
-# type: COUNTER
-# # "__execute-time-ns/pulsar-prod-4/default": "418764",
-# - pattern: __(?!jvm-+)(.+-count|.+-latency|.+-count|.+-time-ns)/(.+)/(.+)
-# name: $1
-# attrNameSnakeCase: true
-# type: COUNTER
-# labels:
-# dest_component: "$2"
-# context: "$3"
-# # "__execute-time-ns/pulsar-prod-4": "418764",
-# - pattern: __(?!jvm-+)(.+-count|.+-latency|.+-count|.+-time-ns)/(.+)
-# name: $1
-# attrNameSnakeCase: true
-# type: COUNTER
-# labels:
-# context: "$2"
-# # StreamManager
-# # "__client_stmgr-17/__bytes_to_stmgrs": "7841039",
-# - pattern: __(client_stmgr-.+)/__(.+_to_stmgrs)
-# name: $2
-# attrNameSnakeCase: true
-# type: COUNTER
-# labels:
-# dest_component: "$1"
-# # "__connection_buffer_by_instanceid/container_1_pulsar-prod-9_201/bytes": "0.000000",
-# - pattern: __(connection_buffer_by_instanceid)/container_(.+)_(.+)/(.+)
-# name: $1_$4
-# attrNameSnakeCase: true
-# type: COUNTER
-# labels:
-# dest_container: "$2"
-# dest_task: "$3"
-# # "__time_spent_back_pressure_by_compid/container_1_pulsar-prod-5_151": "0",
-# - pattern: __(time_spent_back_pressure_by_compid)/container_(.+)_(.+)
-# name: $1
-# attrNameSnakeCase: true
-# type: COUNTER
-# labels:
-# dest_container: "$2"
-# dest_task: "$3"
-# # PulsarSpoutMetrics of PulsarSpout 'PulsarSpoutMetrics/pulsar-prod-4-0/consumerThroughput'
-# - pattern: PulsarSpout/(.+)/(.+)
-# name: pulsar_spout_$2
-# attrNameSnakeCase: true
-# type: COUNTER
-# labels:
-# component: "$1"
-# - pattern: PulsarBolt/(.+)/(.+)
-# name: pulsar_bolt_$2
-# attrNameSnakeCase: true
-# type: COUNTER
-# labels:
-# component: "$1"
-# # name: "kafkaConsumer-request-size-max/consumer-node-metrics/client-id-spout/node-id-node-1"
-# - pattern: kafkaConsumer-(.+)/consumer-(node)-metrics/client-id-(.+)/node-id-(.+)
-# name: kafka_consumer_$2_$1
-# attrNameSnakeCase: true
-# type: COUNTER
-# labels:
-# client_id: "$3"
-# node_id: "$4"
-# # name: "kafkaConsumer-commit-rate/consumer-coordinator-metrics/client-id-spout"
-# - pattern: kafkaConsumer-(.+)/consumer-(coordinator)-metrics/client-id-(.+)
-# name: kafka_consumer_$2_$1
-# attrNameSnakeCase: true
-# type: COUNTER
-# labels:
-# client_id: "$3"
-# # name: "kafkaConsumer-records-lag-max/consumer-fetch-manager-metrics/client-id-spout/topic-nginx-lad-es/partition-1"
-# - pattern: kafkaConsumer-(.+)/consumer-(fetch-manager)-metrics/client-id-(.+)/topic-(.+)/partition-(.+)
-# name: kafka_consumer_$2_$1
-# attrNameSnakeCase: true
-# type: COUNTER
-# labels:
-# client_id: "$3"
-# topic: "$4"
-# partition: "$5"
-# # name: "kafkaConsumer-records-per-request-avg/consumer-fetch-manager-metrics/client-id-spout/topic-nginx-adp-cms-api"
-# - pattern: kafkaConsumer-(.+)/consumer-(fetch-manager)-metrics/client-id-(.+)/topic-(.+)
-# name: kafka_consumer_$2_$1
-# attrNameSnakeCase: true
-# type: COUNTER
-# labels:
-# client_id: "$3"
-# topic: "$4"
-# # name: "kafkaConsumer-bytes-consumed-total/consumer-fetch-manager-metrics/client-id-consumer-1"
-# - pattern: kafkaConsumer-(.+)/consumer-(feath-manager)-metrics/client-id-(.+)
-# name: kafka_consumer_$2_$1
-# attrNameSnakeCase: true
-# type: COUNTER
-# labels:
-# client_id: "$3"
-# - pattern: kafkaConsumer-(.+)/consumer-metrics/client-id-(.+)/node-id-(.+)
-# name: kafka_consumer_$1
-# attrNameSnakeCase: true
-# type: COUNTER
-# labels:
-# client_id: "$2"
-# node_id: "$3"
-# - pattern: kafkaConsumer-(.+)/consumer-metrics/client-id-(.+)
-# name: kafka_consumer_$1
-# attrNameSnakeCase: true
-# type: COUNTER
-# labels:
-# client_id: "$2"
-# - pattern: kafkaConsumer-(.+)/app-info/client-id-(.+)
-# name: kafka_consumer_$1
-# attrNameSnakeCase: true
-# type: COUNTER
-# labels:
-# client_id: "$2"
-# # kafkaOffset of KafkaSpout 'kafkaOffset/topicName/partition_2/spoutLag'
-# - pattern: kafkaOffset/(.+)/partition_([0-9]+)/(.+)
-# name: kafka_offset_$3
-# attrNameSnakeCase: true
-# type: COUNTER
-# labels:
-# topic: "$1"
-# partition: "$2"
-# # kafkaOffset of KafkaSpout 'kafkaOffset/topicName/totalSpoutLag'
-# - pattern: kafkaOffset/(.+)/(.+)
-# name: kafka_offset_$2
-# attrNameSnakeCase: true
-# type: COUNTER
-# labels:
-# topic: "$1"
-
-### Config for graphite-sink
-### Currently the graphite-sink is disabled
-# graphite-sink:
-# class: "org.apache.heron.metricsmgr.sink.GraphiteSink"
-# flush-frequency-ms: 60000
-# graphite_host: "127.0.0.1" # The host of graphite to be exported metrics to
-# graphite_port: 2004 # The port of graphite to be exported metrics to
-# metrics_prefix: "heron" # The prefix of every metrics
-# server_max_reconnect-attempts: 20 # The max reconnect attempts when failing to connect to graphite server
diff --git a/heron/config/src/yaml/conf/aurora/packing.yaml b/heron/config/src/yaml/conf/aurora/packing.yaml
deleted file mode 100644
index 4797a6a0218..00000000000
--- a/heron/config/src/yaml/conf/aurora/packing.yaml
+++ /dev/null
@@ -1,20 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-# packing algorithm for packing instances into containers
-heron.class.packing.algorithm: org.apache.heron.packing.roundrobin.RoundRobinPacking
-heron.class.repacking.algorithm: org.apache.heron.packing.roundrobin.RoundRobinPacking
diff --git a/heron/config/src/yaml/conf/aurora/scheduler.yaml b/heron/config/src/yaml/conf/aurora/scheduler.yaml
deleted file mode 100644
index a3a7a5a53de..00000000000
--- a/heron/config/src/yaml/conf/aurora/scheduler.yaml
+++ /dev/null
@@ -1,61 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-# scheduler class for distributing the topology for execution
-heron.class.scheduler: org.apache.heron.scheduler.aurora.AuroraScheduler
-
-# launcher class for submitting and launching the topology
-heron.class.launcher: org.apache.heron.scheduler.aurora.AuroraLauncher
-
-# location of java - pick it up from shell environment
-heron.directory.sandbox.java.home: /usr/lib/jvm/java-1.8.0-openjdk-amd64/
-
-# Invoke the IScheduler as a library directly
-heron.scheduler.is.service: False
-
-####################################################################
-# Following are Aurora-specific
-####################################################################
-# The maximum retry attempts when trying to kill an Aurora job
-heron.scheduler.job.max.kill.attempts: 5
-
-# The interval in ms between two retry-attempts to kill an Aurora job
-heron.scheduler.job.kill.retry.interval.ms: 2000
-
-# Aurora Controller Class
-# heron.class.scheduler.aurora.controller.cli: False
-
-####################################################################
-# Following are config for tunneling
-####################################################################
-# Whether we should attempt to tunnel if there is no direct access to a remote host (e.g. TManager)
-heron.scheduler.is.tunnel.needed: False
-
-# The connection timeout in ms when testing if we can connect to remote host
-heron.scheduler.tunnel.connection.timeout.ms: 1000
-
-# The number of retries when testing direct access to remote host
-heron.scheduler.tunnel.connection.retry.count: 2
-
-# The interval in ms between two retries when testing direct access to remote host
-heron.scheduler.tunnel.retry.interval.ms: 1000
-
-# The number of retries when verifying connectivity via tunnel
-heron.scheduler.tunnel.verify.count: 10
-
-# SSH tunnel host
-heron.scheduler.tunnel.host: "my.tunnel.host"
diff --git a/heron/config/src/yaml/conf/aurora/stateful.yaml b/heron/config/src/yaml/conf/aurora/stateful.yaml
deleted file mode 100644
index 76e1b0b542b..00000000000
--- a/heron/config/src/yaml/conf/aurora/stateful.yaml
+++ /dev/null
@@ -1,38 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-# indicate the type of storage for checkpoint is HDFS
-heron.statefulstorage.classname: "org.apache.heron.statefulstorage.hdfs.HDFSStorage"
-
-heron.statefulstorage.config:
- heron.statefulstorage.classpath: "$(hadoop --config /path/to/configs classpath)"
- heron.statefulstorage.hdfs.root.path: "/user/heron/checkpoints"
-
-# Following are configs for socket between ckptmgr and stateful storage
-heron.ckptmgr.network.write.batch.size.bytes: 32768
-
-heron.ckptmgr.network.write.batch.time.ms: 16
-
-heron.ckptmgr.network.read.batch.size.bytes: 32768
-
-heron.ckptmgr.network.read.batch.time.ms: 16
-
-heron.ckptmgr.network.options.socket.send.buffer.size.bytes: 655360
-
-heron.ckptmgr.network.options.socket.receive.buffer.size.bytes: 655360
-
-heron.ckptmgr.network.options.maximum.packetsize.bytes: 10485760
diff --git a/heron/config/src/yaml/conf/aurora/statemgr.yaml b/heron/config/src/yaml/conf/aurora/statemgr.yaml
deleted file mode 100644
index a2fe0feb761..00000000000
--- a/heron/config/src/yaml/conf/aurora/statemgr.yaml
+++ /dev/null
@@ -1,49 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-# local state manager class for managing state in a persistent fashion
-heron.class.state.manager: org.apache.heron.statemgr.localfs.LocalFileSystemStateManager
-
-# local state manager connection string
-heron.statemgr.connection.string: LOCALMODE
-
-# path of the root address to store the state in a local file system
-heron.statemgr.root.path: /vagrant/.herondata/repository/state/${CLUSTER}
-
-# create the sub directories, if needed
-heron.statemgr.localfs.is.initialize.file.tree: True
-
-####################################################################
-# Following are config for tunneling
-####################################################################
-# Whether we should attempt to tunnel if there is no direct access to a remote host (e.g. Zookeeper)
-heron.statemgr.is.tunnel.needed: False
-
-# The connection timeout in ms when testing if we can connect to remote host
-heron.statemgr.tunnel.connection.timeout.ms: 1000
-
-# The number of retries when testing direct access to remote host
-heron.statemgr.tunnel.connection.retry.count: 2
-
-# The interval in ms between two retries when testing direct access to remote host
-heron.statemgr.tunnel.retry.interval.ms: 1000
-
-# The number of retries when verifying connectivity via tunnel
-heron.statemgr.tunnel.verify.count: 10
-
-# SSH tunnel host
-heron.statemgr.tunnel.host: "my.tunnel.host"
diff --git a/heron/config/src/yaml/conf/aurora/uploader.yaml b/heron/config/src/yaml/conf/aurora/uploader.yaml
deleted file mode 100644
index 48a6ee48cd1..00000000000
--- a/heron/config/src/yaml/conf/aurora/uploader.yaml
+++ /dev/null
@@ -1,22 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-# uploader class for transferring the topology jar/tar files to storage
-heron.class.uploader: org.apache.heron.uploader.localfs.LocalFileSystemUploader
-
-# name of the directory to upload topologies for local file system uploader
-heron.uploader.localfs.file.system.directory: /vagrant/.herondata/repository/topologies/${CLUSTER}/${ROLE}/${TOPOLOGY}
\ No newline at end of file
diff --git a/heron/config/src/yaml/conf/examples/README.md b/heron/config/src/yaml/conf/examples/README.md
index 96c4e5d65d2..944ebb25ed3 100644
--- a/heron/config/src/yaml/conf/examples/README.md
+++ b/heron/config/src/yaml/conf/examples/README.md
@@ -21,9 +21,6 @@ for different components.
The files are
- * aurora_scheduler.yaml
- - this contains the configuration for aurora scheduler
-
* heron_internals.yaml
- this file contains several configuration elements that govern heron behavior
diff --git a/heron/config/src/yaml/conf/examples/aurora_scheduler.yaml b/heron/config/src/yaml/conf/examples/aurora_scheduler.yaml
deleted file mode 100644
index bef2cfca00a..00000000000
--- a/heron/config/src/yaml/conf/examples/aurora_scheduler.yaml
+++ /dev/null
@@ -1,29 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-# scheduler class for distributing the topology for execution
-heron.class.scheduler: org.apache.heron.scheduler.aurora.AuroraScheduler
-
-# launcher class for submitting and launching the topology
-heron.class.launcher: org.apache.heron.scheduler.aurora.AuroraLauncher
-
-# location of java - pick it up from shell environment
-heron.directory.sandbox.java.home: /usr/lib/jvm/java-1.8.0-openjdk-amd64/
-
-# Invoke the IScheduler as a library directly
-heron.scheduler.is.service: False
-