You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@heron.apache.org by nl...@apache.org on 2019/12/04 00:01:17 UTC
[incubator-heron] branch neng/py3-migration created (now 6a2a4f5)
This is an automated email from the ASF dual-hosted git repository.
nlu90 pushed a change to branch neng/py3-migration
in repository https://gitbox.apache.org/repos/asf/incubator-heron.git.
at 6a2a4f5 migrate to python3 using 2to3 cmd tool
This branch includes the following new commits:
new 6a2a4f5 migrate to python3 using 2to3 cmd tool
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[incubator-heron] 01/01: migrate to python3 using 2to3 cmd tool
Posted by nl...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
nlu90 pushed a commit to branch neng/py3-migration
in repository https://gitbox.apache.org/repos/asf/incubator-heron.git
commit 6a2a4f541522cc9f597c3c5dd99ee373a4adb260
Author: Neng Lu <nl...@twitter.com>
AuthorDate: Tue Dec 3 15:57:58 2019 -0800
migrate to python3 using 2to3 cmd tool
---
bazel_configure.py | 16 +++----
examples/src/python/bolt/stateful_count_bolt.py | 2 +-
examples/src/python/spout/stateful_word_spout.py | 2 +-
heron/executor/src/python/heron_executor.py | 36 +++++++-------
.../tests/python/heron_executor_unittest.py | 52 ++++++++++----------
heron/instance/src/python/basics/__init__.py | 4 +-
heron/instance/src/python/basics/bolt_instance.py | 4 +-
heron/instance/src/python/basics/spout_instance.py | 4 +-
.../instance/src/python/network/gateway_looper.py | 4 +-
.../src/python/utils/metrics/metrics_helper.py | 4 +-
.../instance/src/python/utils/misc/communicator.py | 12 ++---
.../python/utils/misc/custom_grouping_helper.py | 2 +-
heron/instance/src/python/utils/system_config.py | 2 +-
.../python/utils/topology/topology_context_impl.py | 2 +-
heron/instance/tests/python/mock_protobuf.py | 2 +-
.../tests/python/utils/communicator_unittest.py | 4 +-
heron/instance/tests/python/utils/log_unittest.py | 8 ++--
.../tests/python/utils/metrics_helper_unittest.py | 2 +-
.../instance/tests/python/utils/mock_generator.py | 4 +-
heron/shell/src/python/handlers/__init__.py | 24 +++++-----
.../src/python/handlers/killexecutorhandler.py | 4 +-
heron/shell/src/python/utils.py | 2 +-
heron/statemgrs/src/python/config.py | 4 +-
heron/statemgrs/src/python/configloader.py | 2 +-
heron/statemgrs/src/python/filestatemanager.py | 9 ++--
heron/statemgrs/src/python/statemanager.py | 4 +-
heron/statemgrs/src/python/zkstatemanager.py | 2 +-
.../tests/python/configloader_unittest.py | 14 +++---
.../tests/python/statemanagerfactory_unittest.py | 2 +-
heron/tools/admin/src/python/main.py | 6 +--
heron/tools/admin/src/python/standalone.py | 14 +++---
heron/tools/cli/src/python/cli_helper.py | 2 +-
heron/tools/cli/src/python/config.py | 14 +++---
heron/tools/cli/src/python/execute.py | 4 +-
heron/tools/cli/src/python/help.py | 2 +-
heron/tools/cli/src/python/main.py | 10 ++--
heron/tools/cli/src/python/opts.py | 2 +-
heron/tools/cli/src/python/result.py | 2 +-
heron/tools/cli/src/python/submit.py | 6 +--
heron/tools/cli/src/python/version.py | 4 +-
heron/tools/common/src/python/access/__init__.py | 4 +-
heron/tools/common/src/python/access/heron_api.py | 16 +++----
.../common/src/python/access/tracker_access.py | 2 +-
heron/tools/common/src/python/utils/config.py | 8 ++--
heron/tools/explorer/src/python/clusters.py | 2 +-
heron/tools/explorer/src/python/help.py | 2 +-
heron/tools/explorer/src/python/logicalplan.py | 14 +++---
heron/tools/explorer/src/python/main.py | 2 +-
heron/tools/explorer/src/python/physicalplan.py | 26 +++++-----
heron/tools/explorer/src/python/topologies.py | 24 +++++-----
heron/tools/tracker/src/python/config.py | 4 +-
.../tools/tracker/src/python/handlers/__init__.py | 56 +++++++++++-----------
.../src/python/handlers/logicalplanhandler.py | 4 +-
.../src/python/handlers/runtimestatehandler.py | 2 +-
heron/tools/tracker/src/python/javaobj.py | 8 ++--
heron/tools/tracker/src/python/main.py | 4 +-
heron/tools/tracker/src/python/query_operators.py | 54 ++++++++++-----------
heron/tools/tracker/src/python/topology.py | 8 ++--
heron/tools/tracker/src/python/tracker.py | 10 ++--
.../tools/tracker/tests/python/tracker_unittest.py | 10 ++--
heron/tools/ui/src/python/args.py | 6 +--
heron/tools/ui/src/python/handlers/api/topology.py | 12 ++---
heron/tools/ui/src/python/handlers/mainhandler.py | 2 +-
heron/tools/ui/src/python/handlers/ranges.py | 2 +-
heron/tools/ui/src/python/handlers/topology.py | 4 +-
heron/tools/ui/src/python/main.py | 2 +-
heronpy/api/cloudpickle.py | 26 +++++-----
heronpy/api/component/component_spec.py | 10 ++--
heronpy/api/metrics.py | 4 +-
heronpy/api/serializer.py | 2 +-
heronpy/api/tests/python/component_unittest.py | 2 +-
heronpy/api/tests/python/metrics_unittest.py | 8 ++--
heronpy/api/topology.py | 10 ++--
heronpy/streamlet/impl/contextimpl.py | 2 +-
heronpy/streamlet/impl/joinbolt.py | 6 +--
heronpy/streamlet/impl/reducebykeyandwindowbolt.py | 2 +-
integration_test/src/python/http_server/main.py | 4 +-
.../integration_test/core/aggregator_bolt.py | 6 +--
.../integration_test/core/integration_test_bolt.py | 2 +-
.../integration_test/core/test_topology_builder.py | 10 ++--
.../src/python/local_test_runner/test_template.py | 4 +-
integration_test/src/python/test_runner/main.py | 16 +++----
.../src/python/topology_test_runner/main.py | 12 ++---
scripts/shutils/save-logs.py | 10 ++--
third_party/python/cpplint/cpplint.py | 44 ++++++++---------
85 files changed, 379 insertions(+), 388 deletions(-)
diff --git a/bazel_configure.py b/bazel_configure.py
index 02e67f1..b82f08e 100755
--- a/bazel_configure.py
+++ b/bazel_configure.py
@@ -144,7 +144,7 @@ def real_program_path(program_name):
return None
def fail(message):
- print("\nFAILED: %s" % message)
+ print(("\nFAILED: %s" % message))
sys.exit(1)
# Assumes the version is at the end of the first line consisting of digits and dots
@@ -277,7 +277,7 @@ variable to specify the full path to yours.'""" % (program, program, program, en
version = assert_min_version(VALUE, min_version)
print_value = "%s (%s)" % (VALUE, version)
- print('Using %s:\t%s' % (msg.ljust(20), print_value))
+ print(('Using %s:\t%s' % (msg.ljust(20), print_value)))
return VALUE
def discover_jdk():
@@ -290,7 +290,7 @@ def discover_jdk():
"You can set the JAVA_HOME environment variavle to specify the full path to yours.")
jdk_bin_path = os.path.dirname(javac_path)
jdk_path = os.path.dirname(jdk_bin_path)
- print('Using %s:\t%s' % ('JDK'.ljust(20), jdk_path))
+ print(('Using %s:\t%s' % ('JDK'.ljust(20), jdk_path)))
return jdk_path
######################################################################
@@ -307,9 +307,9 @@ def discover_tool_default(program, msg, envvar, defvalue):
VALUE = discover_program(program, envvar)
if not VALUE:
VALUE = defvalue
- print('%s:\tnot found, but ok' % (program.ljust(26)))
+ print(('%s:\tnot found, but ok' % (program.ljust(26))))
else:
- print('Using %s:\t%s' % (msg.ljust(20), VALUE))
+ print(('Using %s:\t%s' % (msg.ljust(20), VALUE)))
return VALUE
def export_env_to_file(out_file, env):
@@ -343,7 +343,7 @@ def write_env_exec_file(platform, environ):
out_file.write('$*')
make_executable(env_exec_file)
- print('Wrote the environment exec file %s' % (env_exec_file))
+ print(('Wrote the environment exec file %s' % (env_exec_file)))
######################################################################
@@ -380,7 +380,7 @@ def write_heron_config_header(config_file):
out_file.write(define_string('GIT_BRANCH', discover_git_branch()))
out_file.write(generate_system_defines())
out_file.close()
- print('Wrote the heron config header file: \t"%s"' % (config_file))
+ print(('Wrote the heron config header file: \t"%s"' % (config_file)))
######################################################################
# MAIN program that sets up your workspace for bazel
@@ -390,7 +390,7 @@ def main():
# Discover the platform
platform = discover_platform()
- print("Platform %s" % platform)
+ print(("Platform %s" % platform))
# do differently on mac
if platform == "Darwin":
diff --git a/examples/src/python/bolt/stateful_count_bolt.py b/examples/src/python/bolt/stateful_count_bolt.py
index 2f0eb1f..a3ea6b6 100644
--- a/examples/src/python/bolt/stateful_count_bolt.py
+++ b/examples/src/python/bolt/stateful_count_bolt.py
@@ -36,7 +36,7 @@ class StatefulCountBolt(Bolt, StatefulComponent):
self.logger.info("Checkpoint Snapshot recovered : %s" % str(self.recovered_state))
def pre_save(self, checkpoint_id):
- for (k, v) in self.counter.items():
+ for (k, v) in list(self.counter.items()):
self.recovered_state.put(k, v)
self.logger.info("Checkpoint Snapshot %s : %s" % (checkpoint_id, str(self.recovered_state)))
diff --git a/examples/src/python/spout/stateful_word_spout.py b/examples/src/python/spout/stateful_word_spout.py
index b127167..9f368df 100644
--- a/examples/src/python/spout/stateful_word_spout.py
+++ b/examples/src/python/spout/stateful_word_spout.py
@@ -37,7 +37,7 @@ class StatefulWordSpout(Spout, StatefulComponent):
def pre_save(self, checkpoint_id):
# Purely for debugging purposes
- for (k, v) in self.counter.items():
+ for (k, v) in list(self.counter.items()):
self.recovered_state.put(k, v)
self.logger.info("Checkpoint Snapshot %s : %s" % (checkpoint_id, str(self.recovered_state)))
diff --git a/heron/executor/src/python/heron_executor.py b/heron/executor/src/python/heron_executor.py
index 01f59a9..e4b83e1 100755
--- a/heron/executor/src/python/heron_executor.py
+++ b/heron/executor/src/python/heron_executor.py
@@ -237,10 +237,10 @@ class HeronExecutor(object):
self.heron_internals_config_file = parsed_args.heron_internals_config_file
self.override_config_file = parsed_args.override_config_file
self.component_ram_map =\
- map(lambda x: {x.split(':')[0]:
- int(x.split(':')[1])}, parsed_args.component_ram_map.split(','))
+ [{x.split(':')[0]:
+ int(x.split(':')[1])} for x in parsed_args.component_ram_map.split(',')]
self.component_ram_map =\
- functools.reduce(lambda x, y: dict(x.items() + y.items()), self.component_ram_map)
+ functools.reduce(lambda x, y: dict(list(x.items()) + list(y.items())), self.component_ram_map)
# component_jvm_opts_in_base64 itself is a base64-encoding-json-map, which is appended with
# " at the start and end. It also escapes "=" to "&equals" due to aurora limitation
@@ -256,7 +256,7 @@ class HeronExecutor(object):
base64.b64decode(parsed_args.component_jvm_opts.
lstrip('"').rstrip('"').replace('(61)', '=').replace('=', '='))
if component_jvm_opts_in_json != "":
- for (k, v) in json.loads(component_jvm_opts_in_json).items():
+ for (k, v) in list(json.loads(component_jvm_opts_in_json).items()):
# In json, the component name and JVM options are still in base64 encoding
self.component_jvm_opts[base64.b64decode(k)] = base64.b64decode(v)
@@ -366,7 +366,7 @@ class HeronExecutor(object):
parser.add_argument("--is-stateful", required=True)
parser.add_argument("--checkpoint-manager-classpath", required=True)
parser.add_argument("--checkpoint-manager-port", required=True)
- parser.add_argument("--checkpoint-manager-ram", type=long, required=True)
+ parser.add_argument("--checkpoint-manager-ram", type=int, required=True)
parser.add_argument("--stateful-config-file", required=True)
parser.add_argument("--health-manager-mode", required=True)
parser.add_argument("--health-manager-classpath", required=True)
@@ -793,7 +793,7 @@ class HeronExecutor(object):
'--zkhostportlist=%s' % self.state_manager_connection,
'--zkroot=%s' % self.state_manager_root,
'--stmgr_id=%s' % self.stmgr_ids[self.shard],
- '--instance_ids=%s' % ','.join(map(lambda x: x[0], instance_info)),
+ '--instance_ids=%s' % ','.join([x[0] for x in instance_info]),
'--myhost=%s' % self.master_host,
'--data_port=%s' % str(self.master_port),
'--local_data_port=%s' % str(self.tmaster_controller_port),
@@ -958,8 +958,8 @@ class HeronExecutor(object):
def _kill_processes(self, commands):
# remove the command from processes_to_monitor and kill the process
with self.process_lock:
- for command_name, command in commands.items():
- for process_info in self.processes_to_monitor.values():
+ for command_name, command in list(commands.items()):
+ for process_info in list(self.processes_to_monitor.values()):
if process_info.name == command_name:
del self.processes_to_monitor[process_info.pid]
Log.info("Killing %s process with pid %d: %s" %
@@ -978,7 +978,7 @@ class HeronExecutor(object):
Log.info("Start processes")
processes_to_monitor = {}
# First start all the processes
- for (name, command) in commands.items():
+ for (name, command) in list(commands.items()):
p = self._run_process(name, command)
processes_to_monitor[p.pid] = ProcessInfo(p, name, command)
@@ -999,7 +999,7 @@ class HeronExecutor(object):
(pid, status) = os.wait()
with self.process_lock:
- if pid in self.processes_to_monitor.keys():
+ if pid in list(self.processes_to_monitor.keys()):
old_process_info = self.processes_to_monitor[pid]
name = old_process_info.name
command = old_process_info.command
@@ -1061,10 +1061,10 @@ class HeronExecutor(object):
# if the current command has a matching command in the updated commands we keep it
# otherwise we kill it
- for current_name, current_command in current_commands.items():
+ for current_name, current_command in list(current_commands.items()):
# We don't restart tmaster since it watches the packing plan and updates itself. The stream
# manager is restarted just to reset state, but we could update it to do so without a restart
- if current_name in updated_commands.keys() and \
+ if current_name in list(updated_commands.keys()) and \
current_command == updated_commands[current_name] and \
not current_name.startswith('stmgr-'):
commands_to_keep[current_name] = current_command
@@ -1072,8 +1072,8 @@ class HeronExecutor(object):
commands_to_kill[current_name] = current_command
# updated commands not in the keep list need to be started
- for updated_name, updated_command in updated_commands.items():
- if updated_name not in commands_to_keep.keys():
+ for updated_name, updated_command in list(updated_commands.items()):
+ if updated_name not in list(commands_to_keep.keys()):
commands_to_start[updated_name] = updated_command
return commands_to_kill, commands_to_keep, commands_to_start
@@ -1083,8 +1083,8 @@ class HeronExecutor(object):
Then starts new ones required and kills old ones no longer required.
'''
with self.process_lock:
- current_commands = dict(map((lambda process: (process.name, process.command)),
- self.processes_to_monitor.values()))
+ current_commands = dict(list(map((lambda process: (process.name, process.command)),
+ list(self.processes_to_monitor.values()))))
updated_commands = self.get_commands_to_run()
# get the commands to kill, keep and start
@@ -1176,7 +1176,7 @@ def setup(executor):
Log.info('Executor terminated; exiting all process in executor.')
# Kill child processes first and wait for log collection to finish
- for pid in executor.processes_to_monitor.keys():
+ for pid in list(executor.processes_to_monitor.keys()):
os.kill(pid, signal.SIGTERM)
time.sleep(5)
@@ -1192,7 +1192,7 @@ def setup(executor):
sid = os.getsid(pid)
# POSIX prohibits the change of the process group ID of a session leader
- if pid <> sid:
+ if pid != sid:
Log.info('Set up process group; executor becomes leader')
os.setpgrp() # create new process group, become its leader
diff --git a/heron/executor/tests/python/heron_executor_unittest.py b/heron/executor/tests/python/heron_executor_unittest.py
index 7fbd800..715f66e 100644
--- a/heron/executor/tests/python/heron_executor_unittest.py
+++ b/heron/executor/tests/python/heron_executor_unittest.py
@@ -92,7 +92,7 @@ class HeronExecutorTest(unittest.TestCase):
def build_packing_plan(self, instance_distribution):
packing_plan = PackingPlan()
- for container_id in instance_distribution.keys():
+ for container_id in list(instance_distribution.keys()):
container_plan = packing_plan.container_plans.add()
container_plan.id = int(container_id)
for (component_name, global_task_id, component_index) in instance_distribution[container_id]:
@@ -293,11 +293,11 @@ class HeronExecutorTest(unittest.TestCase):
def test_update_packing_plan(self):
self.executor_0.update_packing_plan(self.packing_plan_expected)
- self.assertEquals(self.packing_plan_expected, self.executor_0.packing_plan)
- self.assertEquals({1: "stmgr-1", 7: "stmgr-7"}, self.executor_0.stmgr_ids)
- self.assertEquals(
+ self.assertEqual(self.packing_plan_expected, self.executor_0.packing_plan)
+ self.assertEqual({1: "stmgr-1", 7: "stmgr-7"}, self.executor_0.stmgr_ids)
+ self.assertEqual(
{0: "metricsmgr-0", 1: "metricsmgr-1", 7: "metricsmgr-7"}, self.executor_0.metricsmgr_ids)
- self.assertEquals(
+ self.assertEqual(
{0: "heron-shell-0", 1: "heron-shell-1", 7: "heron-shell-7"}, self.executor_0.heron_shell_ids)
def test_launch_container_0(self):
@@ -315,20 +315,16 @@ class HeronExecutorTest(unittest.TestCase):
monitored_processes = executor.processes_to_monitor
# convert to (pid, name, command)
- found_processes = list(map(lambda process_info:
- (process_info.pid, process_info.name, process_info.command_str),
- executor.processes))
- found_monitored = list(map(lambda pinfo:
- (pinfo[0], pinfo[1].name, pinfo[1].command_str),
- monitored_processes.items()))
+ found_processes = list([(process_info.pid, process_info.name, process_info.command_str) for process_info in executor.processes])
+ found_monitored = list([(pinfo[0], pinfo[1].name, pinfo[1].command_str) for pinfo in list(monitored_processes.items())])
found_processes.sort(key=lambda tuple: tuple[0])
found_monitored.sort(key=lambda tuple: tuple[0])
- print("do_test_commands - found_processes: %s found_monitored: %s" \
- % (found_processes, found_monitored))
- self.assertEquals(found_processes, found_monitored)
+ print(("do_test_commands - found_processes: %s found_monitored: %s" \
+ % (found_processes, found_monitored)))
+ self.assertEqual(found_processes, found_monitored)
- print("do_test_commands - expected_processes: %s monitored_processes: %s" \
- % (expected_processes, monitored_processes))
+ print(("do_test_commands - expected_processes: %s monitored_processes: %s" \
+ % (expected_processes, monitored_processes)))
self.assert_processes(expected_processes, monitored_processes)
def test_change_instance_dist_container_1(self):
@@ -337,18 +333,18 @@ class HeronExecutorTest(unittest.TestCase):
current_commands = self.executor_1.get_commands_to_run()
temp_dict = dict(
- map((lambda process_info: (process_info.name, process_info.command.split(' '))),
- self.expected_processes_container_1))
+ list(map((lambda process_info: (process_info.name, process_info.command.split(' '))),
+ self.expected_processes_container_1)))
current_json = json.dumps(current_commands, sort_keys=True, cls=CommandEncoder).split(' ')
temp_json = json.dumps(temp_dict, sort_keys=True).split(' ')
- print ("current_json: %s" % current_json)
- print ("temp_json: %s" % temp_json)
+ print(("current_json: %s" % current_json))
+ print(("temp_json: %s" % temp_json))
# better test error report
for (s1, s2) in zip(current_json, temp_json):
- self.assertEquals(s1, s2)
+ self.assertEqual(s1, s2)
# update instance distribution
new_packing_plan = self.build_packing_plan(
@@ -360,20 +356,20 @@ class HeronExecutorTest(unittest.TestCase):
commands_to_kill, commands_to_keep, commands_to_start = \
self.executor_1.get_command_changes(current_commands, updated_commands)
- self.assertEquals(['container_1_exclaim1_2', 'stmgr-1'], sorted(commands_to_kill.keys()))
- self.assertEquals(
+ self.assertEqual(['container_1_exclaim1_2', 'stmgr-1'], sorted(commands_to_kill.keys()))
+ self.assertEqual(
['container_1_exclaim1_1', 'container_1_word_3', 'heron-shell-1', 'metricsmgr-1'],
sorted(commands_to_keep.keys()))
- self.assertEquals(['container_1_word_2', 'stmgr-1'], sorted(commands_to_start.keys()))
+ self.assertEqual(['container_1_word_2', 'stmgr-1'], sorted(commands_to_start.keys()))
def assert_processes(self, expected_processes, found_processes):
- self.assertEquals(len(expected_processes), len(found_processes))
+ self.assertEqual(len(expected_processes), len(found_processes))
for expected_process in expected_processes:
self.assert_process(expected_process, found_processes)
def assert_process(self, expected_process, found_processes):
pid = expected_process.pid
self.assertTrue(found_processes[pid])
- self.assertEquals(expected_process.name, found_processes[pid].name)
- self.assertEquals(expected_process.command, found_processes[pid].command_str)
- self.assertEquals(1, found_processes[pid].attempts)
+ self.assertEqual(expected_process.name, found_processes[pid].name)
+ self.assertEqual(expected_process.command, found_processes[pid].command_str)
+ self.assertEqual(1, found_processes[pid].attempts)
diff --git a/heron/instance/src/python/basics/__init__.py b/heron/instance/src/python/basics/__init__.py
index 58be1fb..f8e1609 100644
--- a/heron/instance/src/python/basics/__init__.py
+++ b/heron/instance/src/python/basics/__init__.py
@@ -17,5 +17,5 @@
'''module for basic python heron component'''
__all__ = ['bolt_instance.py', 'spout_instance.py', 'base_instance']
-from bolt_instance import BoltInstance
-from spout_instance import SpoutInstance
+from .bolt_instance import BoltInstance
+from .spout_instance import SpoutInstance
diff --git a/heron/instance/src/python/basics/bolt_instance.py b/heron/instance/src/python/basics/bolt_instance.py
index b158945..972baf0 100644
--- a/heron/instance/src/python/basics/bolt_instance.py
+++ b/heron/instance/src/python/basics/bolt_instance.py
@@ -21,7 +21,7 @@
'''bolt_instance.py: module for base bolt for python topology'''
import time
-import Queue
+import queue
import heronpy.api.api_constants as api_constants
from heronpy.api.state.stateful_component import StatefulComponent
@@ -181,7 +181,7 @@ class BoltInstance(BaseInstance):
while not self.in_stream.is_empty():
try:
tuples = self.in_stream.poll()
- except Queue.Empty:
+ except queue.Empty:
break
if isinstance(tuples, tuple_pb2.HeronTupleSet):
diff --git a/heron/instance/src/python/basics/spout_instance.py b/heron/instance/src/python/basics/spout_instance.py
index 0d1bf92..3b4dccf 100644
--- a/heron/instance/src/python/basics/spout_instance.py
+++ b/heron/instance/src/python/basics/spout_instance.py
@@ -20,7 +20,7 @@
'''spout_instance.py: module for base spout for python topology'''
-import Queue
+import queue
import time
import collections
@@ -185,7 +185,7 @@ class SpoutInstance(BaseInstance):
while not self.in_stream.is_empty():
try:
tuples = self.in_stream.poll()
- except Queue.Empty:
+ except queue.Empty:
break
if isinstance(tuples, tuple_pb2.HeronTupleSet):
diff --git a/heron/instance/src/python/network/gateway_looper.py b/heron/instance/src/python/network/gateway_looper.py
index ca4f3b5..4b13605 100644
--- a/heron/instance/src/python/network/gateway_looper.py
+++ b/heron/instance/src/python/network/gateway_looper.py
@@ -26,7 +26,7 @@ import os
import time
import select
-from event_looper import EventLooper
+from .event_looper import EventLooper
from heron.common.src.python.utils.log import Log
class GatewayLooper(EventLooper):
@@ -83,7 +83,7 @@ class GatewayLooper(EventLooper):
error_lst = []
if self.sock_map is not None:
- for fd, obj in self.sock_map.items():
+ for fd, obj in list(self.sock_map.items()):
is_r = obj.readable()
is_w = obj.writable()
if is_r:
diff --git a/heron/instance/src/python/utils/metrics/metrics_helper.py b/heron/instance/src/python/utils/metrics/metrics_helper.py
index 6762ebb..39555ba 100644
--- a/heron/instance/src/python/utils/metrics/metrics_helper.py
+++ b/heron/instance/src/python/utils/metrics/metrics_helper.py
@@ -41,7 +41,7 @@ class BaseMetricsHelper(object):
def register_metrics(self, metrics_collector, interval):
"""Registers its metrics to a given metrics collector with a given interval"""
- for field, metrics in self.metrics.items():
+ for field, metrics in list(self.metrics.items()):
metrics_collector.register_metric(field, metrics, interval)
def update_count(self, name, incr_by=1, key=None):
@@ -379,7 +379,7 @@ class MetricsCollector(object):
if metric_value is None:
return
elif isinstance(metric_value, dict):
- for key, value in metric_value.items():
+ for key, value in list(metric_value.items()):
if key is not None and value is not None:
self._add_data_to_message(message, name + "/" + str(key), value)
self._add_data_to_message(message, "%s/%s" % (name, str(key)), value)
diff --git a/heron/instance/src/python/utils/misc/communicator.py b/heron/instance/src/python/utils/misc/communicator.py
index f8eb5ad..c483696 100644
--- a/heron/instance/src/python/utils/misc/communicator.py
+++ b/heron/instance/src/python/utils/misc/communicator.py
@@ -20,7 +20,7 @@
'''communicator.py: module responsible for communication between Python heron modules'''
import sys
-import Queue
+import queue
from heron.common.src.python.utils.log import Log
@@ -40,7 +40,7 @@ class HeronCommunicator(object):
"""
self._producer_callback = producer_cb
self._consumer_callback = consumer_cb
- self._buffer = Queue.Queue()
+ self._buffer = queue.Queue()
self.capacity = sys.maxsize
def register_capacity(self, capacity):
@@ -72,9 +72,9 @@ class HeronCommunicator(object):
if self._producer_callback is not None:
self._producer_callback()
return ret
- except Queue.Empty:
+ except queue.Empty:
Log.debug("%s: Empty in poll()" % str(self))
- raise Queue.Empty
+ raise queue.Empty
def offer(self, item):
"""Offer to the buffer
@@ -87,9 +87,9 @@ class HeronCommunicator(object):
if self._consumer_callback is not None:
self._consumer_callback()
return True
- except Queue.Full:
+ except queue.Full:
Log.debug("%s: Full in offer()" % str(self))
- raise Queue.Full
+ raise queue.Full
def clear(self):
"""Clear the buffer"""
diff --git a/heron/instance/src/python/utils/misc/custom_grouping_helper.py b/heron/instance/src/python/utils/misc/custom_grouping_helper.py
index 64b5a1a..18819d4 100644
--- a/heron/instance/src/python/utils/misc/custom_grouping_helper.py
+++ b/heron/instance/src/python/utils/misc/custom_grouping_helper.py
@@ -45,7 +45,7 @@ class CustomGroupingHelper(object):
def prepare(self, context):
"""Prepares the custom grouping for this component"""
- for stream_id, targets in self.targets.items():
+ for stream_id, targets in list(self.targets.items()):
for target in targets:
target.prepare(context, stream_id)
diff --git a/heron/instance/src/python/utils/system_config.py b/heron/instance/src/python/utils/system_config.py
index 7cd251a..6327578 100644
--- a/heron/instance/src/python/utils/system_config.py
+++ b/heron/instance/src/python/utils/system_config.py
@@ -28,7 +28,7 @@ sys_config = {}
def merge(default, override):
if isinstance(default, dict) and isinstance(override, dict):
- for k, v in override.items():
+ for k, v in list(override.items()):
Log.info("Add overriding configuration '%s'", k)
if k not in default:
default[k] = v
diff --git a/heron/instance/src/python/utils/topology/topology_context_impl.py b/heron/instance/src/python/utils/topology/topology_context_impl.py
index 9824069..95687a0 100644
--- a/heron/instance/src/python/utils/topology/topology_context_impl.py
+++ b/heron/instance/src/python/utils/topology/topology_context_impl.py
@@ -110,7 +110,7 @@ class TopologyContextImpl(TopologyContext):
def get_component_tasks(self, component_id):
"""Returns the task ids allocated for the given component id"""
ret = []
- for task_id, comp_id in self.task_to_component_map.items():
+ for task_id, comp_id in list(self.task_to_component_map.items()):
if comp_id == component_id:
ret.append(task_id)
return ret
diff --git a/heron/instance/tests/python/mock_protobuf.py b/heron/instance/tests/python/mock_protobuf.py
index 13e6d4d..13ed8e3 100644
--- a/heron/instance/tests/python/mock_protobuf.py
+++ b/heron/instance/tests/python/mock_protobuf.py
@@ -38,7 +38,7 @@ def get_mock_config(config_dict=None):
proto_config = topology_pb2.Config()
config_serializer = PythonSerializer()
assert isinstance(config_dict, dict)
- for key, value in config_dict.items():
+ for key, value in list(config_dict.items()):
if isinstance(value, bool):
kvs = proto_config.kvs.add()
kvs.key = key
diff --git a/heron/instance/tests/python/utils/communicator_unittest.py b/heron/instance/tests/python/utils/communicator_unittest.py
index 2865f2f..2b16488 100644
--- a/heron/instance/tests/python/utils/communicator_unittest.py
+++ b/heron/instance/tests/python/utils/communicator_unittest.py
@@ -20,7 +20,7 @@
# pylint: disable=missing-docstring
-import Queue
+import queue
import unittest
from heron.instance.src.python.utils.misc import HeronCommunicator
@@ -41,7 +41,7 @@ class CommunicatorTest(unittest.TestCase):
def test_empty(self):
communicator = HeronCommunicator(producer_cb=None, consumer_cb=None)
- with self.assertRaises(Queue.Empty):
+ with self.assertRaises(queue.Empty):
communicator.poll()
def test_producer_callback(self):
diff --git a/heron/instance/tests/python/utils/log_unittest.py b/heron/instance/tests/python/utils/log_unittest.py
index daf6ccf..d2f9fb9 100644
--- a/heron/instance/tests/python/utils/log_unittest.py
+++ b/heron/instance/tests/python/utils/log_unittest.py
@@ -32,20 +32,20 @@ class LogTest(unittest.TestCase):
pass
def test_stream_process_stdout(self):
- ret = StringIO(u'hello\nworld\n')
+ ret = StringIO('hello\nworld\n')
log_fn = Mock()
with patch("subprocess.Popen") as mock_process:
mock_process.stdout = ret
proc.stream_process_stdout(mock_process, log_fn)
- log_fn.assert_has_calls([call(u'hello\n'), call(u'world\n')])
+ log_fn.assert_has_calls([call('hello\n'), call('world\n')])
def test_async_stream_process_stdout(self):
- ret = StringIO(u'hello\nworld\n')
+ ret = StringIO('hello\nworld\n')
log_fn = Mock()
with patch("subprocess.Popen") as mock_process:
mock_process.stdout = ret
thread = proc.async_stream_process_stdout(mock_process, log_fn)
thread.join()
- log_fn.assert_has_calls([call(u'hello\n'), call(u'world\n')])
+ log_fn.assert_has_calls([call('hello\n'), call('world\n')])
diff --git a/heron/instance/tests/python/utils/metrics_helper_unittest.py b/heron/instance/tests/python/utils/metrics_helper_unittest.py
index 7e81c8d..ad22024 100644
--- a/heron/instance/tests/python/utils/metrics_helper_unittest.py
+++ b/heron/instance/tests/python/utils/metrics_helper_unittest.py
@@ -45,7 +45,7 @@ class BaseMetricsHelperTest(unittest.TestCase):
def test_register_metrics(self):
self.metrics_helper.register_metrics(self.metrics_collector, 60)
- for name, metric in self.metrics.items():
+ for name, metric in list(self.metrics.items()):
self.assertEqual(self.metrics_collector.metrics_map[name], metric)
self.assertEqual(len(self.metrics_collector.time_bucket_in_sec_to_metrics_name[60]), 4)
self.assertIn(60, self.metrics_collector.registered_timers)
diff --git a/heron/instance/tests/python/utils/mock_generator.py b/heron/instance/tests/python/utils/mock_generator.py
index 5ece51d..8809096 100644
--- a/heron/instance/tests/python/utils/mock_generator.py
+++ b/heron/instance/tests/python/utils/mock_generator.py
@@ -93,8 +93,8 @@ def get_a_sample_pplan():
pplan = mock_protobuf.get_mock_pplan(topology=topology, instances=instances)
keys = ["instance_id", "task_id", "comp_index", "comp_name"]
- zipped = zip(instance_ids, task_ids, comp_indexes, comp_names)
- return pplan, [dict(zip(keys, z)) for z in zipped]
+ zipped = list(zip(instance_ids, task_ids, comp_indexes, comp_names))
+ return pplan, [dict(list(zip(keys, z))) for z in zipped]
def make_data_tuple_from_list(lst, serializer=PythonSerializer()):
"""Make HeronDataTuple from a list of objects"""
diff --git a/heron/shell/src/python/handlers/__init__.py b/heron/shell/src/python/handlers/__init__.py
index 264022b..ce39dcc 100644
--- a/heron/shell/src/python/handlers/__init__.py
+++ b/heron/shell/src/python/handlers/__init__.py
@@ -15,15 +15,15 @@
# specific language governing permissions and limitations
# under the License.
''' __init__ '''
-from browsehandler import BrowseHandler
-from downloadhandler import DownloadHandler
-from filedatahandler import FileDataHandler
-from filehandler import FileHandler
-from filestatshandler import FileStatsHandler
-from jmaphandler import JmapHandler
-from jstackhandler import JstackHandler
-from memoryhistogramhandler import MemoryHistogramHandler
-from pmaphandler import PmapHandler
-from pidhandler import PidHandler
-from killexecutorhandler import KillExecutorHandler
-from healthhandler import HealthHandler
+from .browsehandler import BrowseHandler
+from .downloadhandler import DownloadHandler
+from .filedatahandler import FileDataHandler
+from .filehandler import FileHandler
+from .filestatshandler import FileStatsHandler
+from .jmaphandler import JmapHandler
+from .jstackhandler import JstackHandler
+from .memoryhistogramhandler import MemoryHistogramHandler
+from .pmaphandler import PmapHandler
+from .pidhandler import PidHandler
+from .killexecutorhandler import KillExecutorHandler
+from .healthhandler import HealthHandler
diff --git a/heron/shell/src/python/handlers/killexecutorhandler.py b/heron/shell/src/python/handlers/killexecutorhandler.py
index 4f899a0..646a133 100644
--- a/heron/shell/src/python/handlers/killexecutorhandler.py
+++ b/heron/shell/src/python/handlers/killexecutorhandler.py
@@ -23,7 +23,7 @@
import logging
import os
import signal
-import urlparse
+import urllib.parse
import tornado.web
from tornado.options import options
@@ -46,7 +46,7 @@ class KillExecutorHandler(tornado.web.RequestHandler):
logger = logging.getLogger(__file__)
logger.info("Received 'Killing process' request")
- data = dict(urlparse.parse_qsl(self.request.body))
+ data = dict(urllib.parse.parse_qsl(self.request.body))
# check shared secret
sharedSecret = data.get('secret')
diff --git a/heron/shell/src/python/utils.py b/heron/shell/src/python/utils.py
index 5df4cca..ffbe2ce 100644
--- a/heron/shell/src/python/utils.py
+++ b/heron/shell/src/python/utils.py
@@ -182,7 +182,7 @@ def chain(cmd_list):
Feed output of one command to the next and return final output
Returns string output of chained application of commands.
"""
- command = ' | '.join(map(lambda x: ' '.join(x), cmd_list))
+ command = ' | '.join([' '.join(x) for x in cmd_list])
chained_proc = functools.reduce(pipe, [None] + cmd_list)
stdout_builder = proc.async_stdout_builder(chained_proc)
chained_proc.wait()
diff --git a/heron/statemgrs/src/python/config.py b/heron/statemgrs/src/python/config.py
index c06be6c..14975b3 100644
--- a/heron/statemgrs/src/python/config.py
+++ b/heron/statemgrs/src/python/config.py
@@ -46,7 +46,7 @@ class Config(object):
"""
Names of all state locations must be unique.
"""
- names = map(lambda loc: loc["name"], self.locations)
+ names = [loc["name"] for loc in self.locations]
assert len(names) == len(set(names)), "Names of state locations must be unique"
def get_all_state_locations(self):
@@ -59,4 +59,4 @@ class Config(object):
"""
Return a list of state locations of a particular type.
"""
- return filter(lambda location: location['type'] == location_type, self.locations)
+ return [location for location in self.locations if location['type'] == location_type]
diff --git a/heron/statemgrs/src/python/configloader.py b/heron/statemgrs/src/python/configloader.py
index 384a5c7..37e69f7 100644
--- a/heron/statemgrs/src/python/configloader.py
+++ b/heron/statemgrs/src/python/configloader.py
@@ -98,4 +98,4 @@ if __name__ == "__main__":
locations = load_state_manager_locations('local', sys.argv[1])
else:
locations = load_state_manager_locations('local')
- print("locations: %s" % locations)
+ print(("locations: %s" % locations))
diff --git a/heron/statemgrs/src/python/filestatemanager.py b/heron/statemgrs/src/python/filestatemanager.py
index fa620bb..58b2a7b 100644
--- a/heron/statemgrs/src/python/filestatemanager.py
+++ b/heron/statemgrs/src/python/filestatemanager.py
@@ -99,7 +99,7 @@ class FileStateManager(StateManager):
For all the topologies in the watchers, check if the data
in directory has changed. Trigger the callback if it has.
"""
- for topology, callbacks in watchers.items():
+ for topology, callbacks in list(watchers.items()):
file_path = os.path.join(path, topology)
data = ""
if os.path.exists(file_path):
@@ -117,9 +117,7 @@ class FileStateManager(StateManager):
topologies = []
if os.path.isdir(topologies_path):
- topologies = list(filter(
- lambda f: os.path.isfile(os.path.join(topologies_path, f)),
- os.listdir(topologies_path)))
+ topologies = list([f for f in os.listdir(topologies_path) if os.path.isfile(os.path.join(topologies_path, f))])
if set(topologies) != set(self.topologies_directory):
for callback in self.topologies_watchers:
callback(topologies)
@@ -166,8 +164,7 @@ class FileStateManager(StateManager):
self.topologies_watchers.append(callback)
else:
topologies_path = self.get_topologies_path()
- return filter(lambda f: os.path.isfile(os.path.join(topologies_path, f)),
- os.listdir(topologies_path))
+ return [f for f in os.listdir(topologies_path) if os.path.isfile(os.path.join(topologies_path, f))]
def get_topology(self, topologyName, callback=None):
"""get topology"""
diff --git a/heron/statemgrs/src/python/statemanager.py b/heron/statemgrs/src/python/statemanager.py
index 94066f1..a0f7492 100644
--- a/heron/statemgrs/src/python/statemanager.py
+++ b/heron/statemgrs/src/python/statemanager.py
@@ -33,15 +33,13 @@ HERON_TMASTER_PREFIX = "{0}/tmasters/"
HERON_TOPOLOGIES_KEY = "{0}/topologies"
# pylint: disable=too-many-public-methods, attribute-defined-outside-init
-class StateManager:
+class StateManager(metaclass=abc.ABCMeta):
"""
This is the abstract base class for state manager. It provides methods to get/set/delete various
state from the state store. The getters accept an optional callback, which will watch for state
changes of the object and invoke the callback when one occurs.
"""
- __metaclass__ = abc.ABCMeta
-
TIMEOUT_SECONDS = 5
@property
diff --git a/heron/statemgrs/src/python/zkstatemanager.py b/heron/statemgrs/src/python/zkstatemanager.py
index f130ead..438a019 100644
--- a/heron/statemgrs/src/python/zkstatemanager.py
+++ b/heron/statemgrs/src/python/zkstatemanager.py
@@ -40,7 +40,7 @@ from kazoo.exceptions import NotEmptyError
from kazoo.exceptions import ZookeeperError
def _makehostportlist(hostportlist):
- return ','.join(map(lambda hp: "%s:%i" % hp, hostportlist))
+ return ','.join(["%s:%i" % hp for hp in hostportlist])
# pylint: disable=attribute-defined-outside-init
class ZkStateManager(StateManager):
diff --git a/heron/statemgrs/tests/python/configloader_unittest.py b/heron/statemgrs/tests/python/configloader_unittest.py
index bdc381b..3549e66 100644
--- a/heron/statemgrs/tests/python/configloader_unittest.py
+++ b/heron/statemgrs/tests/python/configloader_unittest.py
@@ -40,7 +40,7 @@ class ConfigLoaderTest(unittest.TestCase):
return configloader.load_state_manager_locations(cluster, yaml_path)
def test_load_state_manager_locations_aurora(self):
- self.assertEquals([{
+ self.assertEqual([{
'hostport': 'LOCALMODE',
'name': 'local',
'rootpath': '/vagrant/.herondata/repository/state/aurora',
@@ -49,7 +49,7 @@ class ConfigLoaderTest(unittest.TestCase):
}], self.load_locations('aurora'))
def test_load_state_manager_locations_local(self):
- self.assertEquals([{
+ self.assertEqual([{
'hostport': 'LOCALMODE',
'name': 'local',
'rootpath': '/user/fake/.herondata/repository/state/local',
@@ -58,7 +58,7 @@ class ConfigLoaderTest(unittest.TestCase):
}], self.load_locations('local'))
def test_load_state_manager_locations_localzk(self):
- self.assertEquals([{
+ self.assertEqual([{
'hostport': '127.0.0.1:2181',
'name': 'zk',
'rootpath': '/heron',
@@ -67,7 +67,7 @@ class ConfigLoaderTest(unittest.TestCase):
}], self.load_locations('localzk'))
def test_load_state_manager_locations_marathon(self):
- self.assertEquals([{
+ self.assertEqual([{
'hostport': 'LOCALMODE',
'name': 'local',
'rootpath': '/user/fake/.herondata/repository/state/marathon',
@@ -76,7 +76,7 @@ class ConfigLoaderTest(unittest.TestCase):
}], self.load_locations('marathon'))
def test_load_state_manager_locations_mesos(self):
- self.assertEquals([{
+ self.assertEqual([{
'hostport': 'LOCALMODE',
'name': 'local',
'rootpath': '/user/fake/.herondata/repository/state/mesos',
@@ -85,7 +85,7 @@ class ConfigLoaderTest(unittest.TestCase):
}], self.load_locations('mesos'))
def test_load_state_manager_locations_slurm(self):
- self.assertEquals([{
+ self.assertEqual([{
'hostport': 'LOCALMODE',
'name': 'local',
'rootpath': '/user/fake/.herondata/repository/state/slurm',
@@ -94,7 +94,7 @@ class ConfigLoaderTest(unittest.TestCase):
}], self.load_locations('slurm'))
def test_load_state_manager_locations_yarn(self):
- self.assertEquals([{
+ self.assertEqual([{
'hostport': 'LOCALMODE',
'name': 'local',
'rootpath': '/user/fake/.herondata/repository/state/yarn',
diff --git a/heron/statemgrs/tests/python/statemanagerfactory_unittest.py b/heron/statemgrs/tests/python/statemanagerfactory_unittest.py
index e56c90c..dea8cfe 100644
--- a/heron/statemgrs/tests/python/statemanagerfactory_unittest.py
+++ b/heron/statemgrs/tests/python/statemanagerfactory_unittest.py
@@ -34,7 +34,7 @@ class StateManagerFacotryTest(unittest.TestCase):
'rootpath':'/heron', 'tunnelhost':'127.0.0.1'}])
statemanagers = statemanagerfactory.get_all_zk_state_managers(conf)
# 1 state_location should result in 1 state manager
- self.assertEquals(1, len(statemanagers))
+ self.assertEqual(1, len(statemanagers))
statemanager = statemanagers[0]
# statemanager.hostportlist should contain both host port pairs
diff --git a/heron/tools/admin/src/python/main.py b/heron/tools/admin/src/python/main.py
index 57348b6..cedadec 100644
--- a/heron/tools/admin/src/python/main.py
+++ b/heron/tools/admin/src/python/main.py
@@ -51,9 +51,9 @@ class _HelpAction(argparse._HelpAction):
# but better save than sorry
for subparsers_action in subparsers_actions:
# get all subparsers and print help
- for choice, subparser in subparsers_action.choices.items():
- print("Subparser '{}'".format(choice))
- print(subparser.format_help())
+ for choice, subparser in list(subparsers_action.choices.items()):
+ print(("Subparser '{}'".format(choice)))
+ print((subparser.format_help()))
return
################################################################################
diff --git a/heron/tools/admin/src/python/standalone.py b/heron/tools/admin/src/python/standalone.py
index 85ee341..f418afa 100644
--- a/heron/tools/admin/src/python/standalone.py
+++ b/heron/tools/admin/src/python/standalone.py
@@ -185,11 +185,11 @@ def run(command, parser, cl_args, unknown_args):
elif action == Action.GET:
action_type = cl_args["type"]
if action_type == Get.SERVICE_URL:
- print get_service_url(cl_args)
+ print(get_service_url(cl_args))
elif action_type == Get.HERON_UI_URL:
- print get_heron_ui_url(cl_args)
+ print(get_heron_ui_url(cl_args))
elif action_type == Get.HERON_TRACKER_URL:
- print get_heron_tracker_url(cl_args)
+ print(get_heron_tracker_url(cl_args))
else:
raise ValueError("Invalid get action %s" % action_type)
elif action == Action.INFO:
@@ -312,7 +312,7 @@ def template_file(src, dest, replacements_dict):
file_contents = ""
with open(src, 'r') as tf:
file_contents = tf.read()
- for key, value in replacements_dict.iteritems():
+ for key, value in replacements_dict.items():
file_contents = file_contents.replace(key, value)
if not file_contents:
@@ -372,7 +372,7 @@ def print_cluster_info(cl_args):
info['roles'] = roles
info['urls'] = urls
- print json.dumps(info, indent=2)
+ print(json.dumps(info, indent=2))
def add_additional_args(parsers):
'''
@@ -858,13 +858,13 @@ def get_hostname(ip_addr, cl_args):
return output[0].strip("\n")
def check_sure(cl_args, prompt):
- yes = raw_input("%s" % prompt + ' (yes/no): ')
+ yes = input("%s" % prompt + ' (yes/no): ')
if yes == "y" or yes == "yes":
return True
elif yes == "n" or yes == "no":
return False
else:
- print 'Invalid input. Please input "yes" or "no"'
+ print('Invalid input. Please input "yes" or "no"')
def get_jobs(cl_args, nomad_addr):
r = requests.get("http://%s:4646/v1/jobs" % nomad_addr)
diff --git a/heron/tools/cli/src/python/cli_helper.py b/heron/tools/cli/src/python/cli_helper.py
index 439621d..4b73661 100644
--- a/heron/tools/cli/src/python/cli_helper.py
+++ b/heron/tools/cli/src/python/cli_helper.py
@@ -58,7 +58,7 @@ def create_parser(subparsers, action, help_arg):
################################################################################
def flatten_args(fargs):
temp_args = []
- for k, v in fargs.items():
+ for k, v in list(fargs.items()):
if isinstance(v, list):
temp_args.extend([(k, value) for value in v])
else:
diff --git a/heron/tools/cli/src/python/config.py b/heron/tools/cli/src/python/config.py
index d926ebd..5091b2f 100644
--- a/heron/tools/cli/src/python/config.py
+++ b/heron/tools/cli/src/python/config.py
@@ -91,10 +91,10 @@ def _list(cl_args):
cluster = cl_args['cluster']
config = cliconfig.cluster_config(cluster)
if config:
- for k, v in config.items():
- print("%s = %s" % (str(k), str(v)))
+ for k, v in list(config.items()):
+ print(("%s = %s" % (str(k), str(v))))
else:
- print("No config for cluster %s" % cluster)
+ print(("No config for cluster %s" % cluster))
return SimpleResult(Status.Ok)
@@ -103,9 +103,9 @@ def _set(cl_args):
cluster, prop, value = cl_args['cluster'], cl_args['property'], cl_args['value']
if cliconfig.is_valid_property(prop):
cliconfig.set_property(cluster, prop, value)
- print("Updated property [%s] for cluster %s" % (prop, cluster))
+ print(("Updated property [%s] for cluster %s" % (prop, cluster)))
else:
- print("Error: Unknown property [%s] for cluster %s" % (prop, cluster))
+ print(("Error: Unknown property [%s] for cluster %s" % (prop, cluster)))
return SimpleResult(Status.Ok)
@@ -115,9 +115,9 @@ def _unset(cl_args):
cluster, prop = cl_args['cluster'], cl_args['property']
if cliconfig.is_valid_property(prop):
cliconfig.unset_property(cluster, prop)
- print("Cleared property [%s] for cluster %s" % (prop, cluster))
+ print(("Cleared property [%s] for cluster %s" % (prop, cluster)))
else:
- print("Error: Unknown property [%s] for cluster %s" % (prop, cluster))
+ print(("Error: Unknown property [%s] for cluster %s" % (prop, cluster)))
return SimpleResult(Status.Ok)
diff --git a/heron/tools/cli/src/python/execute.py b/heron/tools/cli/src/python/execute.py
index ba2d982..4a052b4 100644
--- a/heron/tools/cli/src/python/execute.py
+++ b/heron/tools/cli/src/python/execute.py
@@ -160,8 +160,8 @@ def heron_cpp(topology_binary, args=None):
cmd.extend(args)
Log.debug("Invoking binary using command: ``%s''", ' '.join(cmd))
Log.debug('Heron options: {%s}', str(heron_env['HERON_OPTIONS']))
- print("Invoking class using command: ``%s''" % ' '.join(cmd))
- print('Heron options: {%s}' % str(heron_env['HERON_OPTIONS']))
+ print(("Invoking class using command: ``%s''" % ' '.join(cmd)))
+ print(('Heron options: {%s}' % str(heron_env['HERON_OPTIONS'])))
# invoke the command with subprocess and print error message, if any
proc = subprocess.Popen(cmd, env=heron_env, stdout=subprocess.PIPE,
stderr=subprocess.PIPE, bufsize=1)
diff --git a/heron/tools/cli/src/python/help.py b/heron/tools/cli/src/python/help.py
index 6edd8a0..f3dd9dd 100644
--- a/heron/tools/cli/src/python/help.py
+++ b/heron/tools/cli/src/python/help.py
@@ -67,7 +67,7 @@ def run(command, parser, args, unknown_args):
# get the subparser for the specific command
subparser = config.get_subparser(parser, command_help)
if subparser:
- print(subparser.format_help())
+ print((subparser.format_help()))
return SimpleResult(Status.Ok)
else:
Log.error("Unknown subcommand \'%s\'", command_help)
diff --git a/heron/tools/cli/src/python/main.py b/heron/tools/cli/src/python/main.py
index 11b4695..98c7fab 100644
--- a/heron/tools/cli/src/python/main.py
+++ b/heron/tools/cli/src/python/main.py
@@ -69,9 +69,9 @@ class _HelpAction(argparse._HelpAction):
# but better save than sorry
for subparsers_action in subparsers_actions:
# get all subparsers and print help
- for choice, subparser in subparsers_action.choices.items():
- print("Subparser '{}'".format(choice))
- print(subparser.format_help())
+ for choice, subparser in list(subparsers_action.choices.items()):
+ print(("Subparser '{}'".format(choice)))
+ print((subparser.format_help()))
return
################################################################################
@@ -226,7 +226,7 @@ def direct_deployment_mode(command, parser, cluster, cl_args):
except KeyError:
# if some of the arguments are not found, print error and exit
subparser = config.get_subparser(parser, command)
- print(subparser.format_help())
+ print((subparser.format_help()))
return dict()
# check if the cluster config directory exists
@@ -292,7 +292,7 @@ def extract_common_args(command, parser, cl_args):
except KeyError:
# if some of the arguments are not found, print error and exit
subparser = config.get_subparser(parser, command)
- print(subparser.format_help())
+ print((subparser.format_help()))
return dict()
new_cl_args = dict()
diff --git a/heron/tools/cli/src/python/opts.py b/heron/tools/cli/src/python/opts.py
index a15cbc6..be29bf2 100644
--- a/heron/tools/cli/src/python/opts.py
+++ b/heron/tools/cli/src/python/opts.py
@@ -37,7 +37,7 @@ def get_heron_config():
:return:
'''
opt_list = []
- for (key, value) in config_opts.items():
+ for (key, value) in list(config_opts.items()):
opt_list.append('%s=%s' % (key, value))
all_opts = (','.join(opt_list)).replace(' ', '%%%%')
diff --git a/heron/tools/cli/src/python/result.py b/heron/tools/cli/src/python/result.py
index e8b556d..d87eb35 100644
--- a/heron/tools/cli/src/python/result.py
+++ b/heron/tools/cli/src/python/result.py
@@ -19,7 +19,7 @@
# under the License.
'''result.py'''
-from __future__ import print_function
+
import abc
import sys
from enum import Enum
diff --git a/heron/tools/cli/src/python/submit.py b/heron/tools/cli/src/python/submit.py
index f147619..4ff9097 100644
--- a/heron/tools/cli/src/python/submit.py
+++ b/heron/tools/cli/src/python/submit.py
@@ -25,7 +25,7 @@ import os
import tempfile
import requests
import subprocess
-import urlparse
+import urllib.parse
from heron.common.src.python.utils.log import Log
from heron.proto import topology_pb2
@@ -200,7 +200,7 @@ def launch_topology_server(cl_args, topology_file, topology_defn_file, topology_
Log.error(r.json().get('message', "Unknown error from API server %d" % r.status_code))
elif ok:
# this case happens when we request a dry_run
- print(r.json().get("response"))
+ print((r.json().get("response")))
except (requests.exceptions.ConnectionError, requests.exceptions.HTTPError) as err:
Log.error(err)
return SimpleResult(Status.HeronError, err_ctxt, succ_ctxt)
@@ -418,7 +418,7 @@ def run(command, parser, cl_args, unknown_args):
# get the topology file name
topology_file = cl_args['topology-file-name']
- if urlparse.urlparse(topology_file).scheme:
+ if urllib.parse.urlparse(topology_file).scheme:
cl_args['topology-file-name'] = download(topology_file, cl_args['cluster'])
topology_file = cl_args['topology-file-name']
Log.debug("download uri to local file: %s", topology_file)
diff --git a/heron/tools/cli/src/python/version.py b/heron/tools/cli/src/python/version.py
index 1371e48..7f0e87a 100644
--- a/heron/tools/cli/src/python/version.py
+++ b/heron/tools/cli/src/python/version.py
@@ -100,9 +100,9 @@ def run(command, parser, cl_args, unknown_args):
r = service_method(service_apiurl)
if r.status_code != requests.codes.ok:
Log.error(r.json().get('message', "Unknown error from API server %d" % r.status_code))
- sorted_items = sorted(r.json().items(), key=lambda tup: tup[0])
+ sorted_items = sorted(list(r.json().items()), key=lambda tup: tup[0])
for key, value in sorted_items:
- print("%s : %s" % (key, value))
+ print(("%s : %s" % (key, value)))
except (requests.exceptions.ConnectionError, requests.exceptions.HTTPError) as err:
Log.error(err)
return SimpleResult(Status.HeronError)
diff --git a/heron/tools/common/src/python/access/__init__.py b/heron/tools/common/src/python/access/__init__.py
index 8e6a8a6..9639b9f 100644
--- a/heron/tools/common/src/python/access/__init__.py
+++ b/heron/tools/common/src/python/access/__init__.py
@@ -17,5 +17,5 @@
''' access module '''
__all__ = ['access']
-from heron_api import *
-from query import *
+from .heron_api import *
+from .query import *
diff --git a/heron/tools/common/src/python/access/heron_api.py b/heron/tools/common/src/python/access/heron_api.py
index ebf908d..c5a5203 100644
--- a/heron/tools/common/src/python/access/heron_api.py
+++ b/heron/tools/common/src/python/access/heron_api.py
@@ -25,8 +25,8 @@ import logging
import tornado.httpclient
import tornado.gen
from tornado.options import options
-from fetch import fetch_url_as_json
-from query import QueryHandler
+from .fetch import fetch_url_as_json
+from .query import QueryHandler
# pylint: disable=bad-whitespace
CLUSTER_URL_FMT = "%s/clusters"
@@ -243,7 +243,7 @@ def get_comps(cluster, environ, topology, role=None):
request_url = tornado.httputil.url_concat(
create_url(LOGICALPLAN_URL_FMT), params)
lplan = yield fetch_url_as_json(request_url)
- comps = lplan['spouts'].keys() + lplan['bolts'].keys()
+ comps = list(lplan['spouts'].keys()) + list(lplan['bolts'].keys())
raise tornado.gen.Return(comps)
################################################################################
@@ -263,7 +263,7 @@ def get_instances(cluster, environ, topology, role=None):
request_url = tornado.httputil.url_concat(
create_url(PHYSICALPLAN_URL_FMT), params)
pplan = yield fetch_url_as_json(request_url)
- instances = pplan['instances'].keys()
+ instances = list(pplan['instances'].keys())
raise tornado.gen.Return(instances)
@@ -404,7 +404,7 @@ def get_comp_instance_metrics(cluster, environ, topology, component,
all_instances = instances if isinstance(instances, list) else [instances]
# append each metric to the url
- for _, metric_name in metrics.items():
+ for _, metric_name in list(metrics.items()):
request_url = tornado.httputil.url_concat(request_url, dict(metricname=metric_name[0]))
# append each instance to the url
@@ -856,7 +856,7 @@ class HeronQueryHandler(QueryHandler):
timelines.extend(result["timeline"])
result = self.get_metric_response(timerange, timelines, is_max)
else:
- data = self.compute_max(res.values())
+ data = self.compute_max(list(res.values()))
result = self.get_metric_response(timerange, data, is_max)
raise tornado.gen.Return(result)
@@ -871,10 +871,10 @@ class HeronQueryHandler(QueryHandler):
# key set is empty. These components are filtered out first.
filtered_ts = [ts for ts in multi_ts if len(ts["timeline"][0]["data"]) > 0]
if len(filtered_ts) > 0 and len(filtered_ts[0]["timeline"]) > 0:
- keys = filtered_ts[0]["timeline"][0]["data"].keys()
+ keys = list(filtered_ts[0]["timeline"][0]["data"].keys())
timelines = ([res["timeline"][0]["data"][key] for key in keys] for res in filtered_ts)
values = (max(v) for v in zip(*timelines))
- return dict(zip(keys, values))
+ return dict(list(zip(keys, values)))
return {}
# pylint: disable=no-self-use
diff --git a/heron/tools/common/src/python/access/tracker_access.py b/heron/tools/common/src/python/access/tracker_access.py
index 4e5f4b0..8b49b97 100644
--- a/heron/tools/common/src/python/access/tracker_access.py
+++ b/heron/tools/common/src/python/access/tracker_access.py
@@ -48,7 +48,7 @@ def metric_queries():
def queries_map():
"""map from query parameter to query name"""
qs = _all_metric_queries()
- return dict(zip(qs[0], qs[1]) + zip(qs[2], qs[3]))
+ return dict(list(zip(qs[0], qs[1])) + list(zip(qs[2], qs[3])))
def get_clusters():
diff --git a/heron/tools/common/src/python/utils/config.py b/heron/tools/common/src/python/utils/config.py
index e08c403..254bdd9 100644
--- a/heron/tools/common/src/python/utils/config.py
+++ b/heron/tools/common/src/python/utils/config.py
@@ -101,7 +101,7 @@ def get_subparser(parser, command):
# but better save than sorry
for subparsers_action in subparsers_actions:
# get all subparsers
- for choice, subparser in subparsers_action.choices.items():
+ for choice, subparser in list(subparsers_action.choices.items()):
if choice == command:
return subparser
return None
@@ -282,7 +282,7 @@ def parse_cluster_role_env(cluster_role_env, config_path):
if tmp_confs is not None:
cli_confs = tmp_confs
else:
- print("Failed to read: %s due to it is empty" % (CLIENT_YAML))
+ print(("Failed to read: %s due to it is empty" % (CLIENT_YAML)))
# if role is required but not provided, raise exception
if len(parts) == 1:
@@ -467,9 +467,9 @@ def print_build_info(zipped_pex=False):
with open(release_file) as release_info:
release_map = yaml.load(release_info)
- release_items = sorted(release_map.items(), key=lambda tup: tup[0])
+ release_items = sorted(list(release_map.items()), key=lambda tup: tup[0])
for key, value in release_items:
- print("%s : %s" % (key, value))
+ print(("%s : %s" % (key, value)))
def get_version_number(zipped_pex=False):
"""Print version from release.yaml
diff --git a/heron/tools/explorer/src/python/clusters.py b/heron/tools/explorer/src/python/clusters.py
index 4efb25a..e70957a 100644
--- a/heron/tools/explorer/src/python/clusters.py
+++ b/heron/tools/explorer/src/python/clusters.py
@@ -46,5 +46,5 @@ def run(command, parser, cl_args, unknown_args):
return False
print('Available clusters:')
for cluster in clusters:
- print(' %s' % cluster)
+ print((' %s' % cluster))
return True
diff --git a/heron/tools/explorer/src/python/help.py b/heron/tools/explorer/src/python/help.py
index 37edee9..d543379 100644
--- a/heron/tools/explorer/src/python/help.py
+++ b/heron/tools/explorer/src/python/help.py
@@ -58,7 +58,7 @@ def run(command, parser, args, unknown_args):
# get the subparser for the specific command
subparser = config.get_subparser(parser, command_help)
if subparser:
- print(subparser.format_help())
+ print((subparser.format_help()))
return True
else:
Log.error("Unknown subcommand \'%s\'" % command_help)
diff --git a/heron/tools/explorer/src/python/logicalplan.py b/heron/tools/explorer/src/python/logicalplan.py
index c7e4085..e056ff6 100644
--- a/heron/tools/explorer/src/python/logicalplan.py
+++ b/heron/tools/explorer/src/python/logicalplan.py
@@ -62,9 +62,9 @@ def parse_topo_loc(cl_args):
def to_table(components, topo_info):
""" normalize raw logical plan info to table """
inputs, outputs = defaultdict(list), defaultdict(list)
- for ctype, component in components.items():
+ for ctype, component in list(components.items()):
if ctype == 'bolts':
- for component_name, component_info in component.items():
+ for component_name, component_info in list(component.items()):
for input_stream in component_info['inputs']:
input_name = input_stream['component_name']
inputs[component_name].append(input_name)
@@ -72,11 +72,11 @@ def to_table(components, topo_info):
info = []
spouts_instance = topo_info['physical_plan']['spouts']
bolts_instance = topo_info['physical_plan']['bolts']
- for ctype, component in components.items():
+ for ctype, component in list(components.items()):
# stages is an int so keep going
if ctype == "stages":
continue
- for component_name, component_info in component.items():
+ for component_name, component_info in list(component.items()):
row = [ctype[:-1], component_name]
if ctype == 'spouts':
row.append(len(spouts_instance[component_name]))
@@ -118,13 +118,13 @@ def run(cl_args, compo_type):
topo_info = tracker_access.get_topology_info(cluster, env, topology, role)
table, header = to_table(components, topo_info)
if spouts_only == bolts_only:
- print(tabulate(table, headers=header))
+ print((tabulate(table, headers=header)))
elif spouts_only:
table, header = filter_spouts(table, header)
- print(tabulate(table, headers=header))
+ print((tabulate(table, headers=header)))
else:
table, header = filter_bolts(table, header)
- print(tabulate(table, headers=header))
+ print((tabulate(table, headers=header)))
return True
except:
Log.error("Fail to connect to tracker: \'%s\'", cl_args["tracker_url"])
diff --git a/heron/tools/explorer/src/python/main.py b/heron/tools/explorer/src/python/main.py
index 4147e04..e8c75e6 100644
--- a/heron/tools/explorer/src/python/main.py
+++ b/heron/tools/explorer/src/python/main.py
@@ -138,7 +138,7 @@ def extract_common_args(command, parser, cl_args):
except KeyError:
# if some of the arguments are not found, print error and exit
subparser = config.get_subparser(parser, command)
- print(subparser.format_help())
+ print((subparser.format_help()))
return dict()
cluster = config.get_heron_cluster(cluster_role_env)
config_path = config.get_heron_cluster_conf_dir(cluster, config_path)
diff --git a/heron/tools/explorer/src/python/physicalplan.py b/heron/tools/explorer/src/python/physicalplan.py
index 72c5dfa..96fa59a 100644
--- a/heron/tools/explorer/src/python/physicalplan.py
+++ b/heron/tools/explorer/src/python/physicalplan.py
@@ -76,7 +76,7 @@ def to_table(metrics):
""" normalize raw metrics API result to table """
all_queries = tracker_access.metric_queries()
m = tracker_access.queries_map()
- names = metrics.values()[0].keys()
+ names = list(metrics.values())[0].keys()
stats = []
for n in names:
info = [n]
@@ -86,7 +86,7 @@ def to_table(metrics):
except KeyError:
pass
stats.append(info)
- header = ['container id'] + [m[k] for k in all_queries if k in metrics.keys()]
+ header = ['container id'] + [m[k] for k in all_queries if k in list(metrics.keys())]
return stats, header
@@ -97,8 +97,8 @@ def run_metrics(command, parser, cl_args, unknown_args):
topology = cl_args['topology-name']
try:
result = tracker_access.get_topology_info(cluster, env, topology, role)
- spouts = result['physical_plan']['spouts'].keys()
- bolts = result['physical_plan']['bolts'].keys()
+ spouts = list(result['physical_plan']['spouts'].keys())
+ bolts = list(result['physical_plan']['bolts'].keys())
components = spouts + bolts
cname = cl_args['component']
if cname:
@@ -122,8 +122,8 @@ def run_metrics(command, parser, cl_args, unknown_args):
for i, (comp, stat, header) in enumerate(cresult):
if i != 0:
print('')
- print('\'%s\' metrics:' % comp)
- print(tabulate(stat, headers=header))
+ print(('\'%s\' metrics:' % comp))
+ print((tabulate(stat, headers=header)))
return True
@@ -134,7 +134,7 @@ def run_bolts(command, parser, cl_args, unknown_args):
topology = cl_args['topology-name']
try:
result = tracker_access.get_topology_info(cluster, env, topology, role)
- bolts = result['physical_plan']['bolts'].keys()
+ bolts = list(result['physical_plan']['bolts'].keys())
bolt_name = cl_args['bolt']
if bolt_name:
if bolt_name in bolts:
@@ -157,8 +157,8 @@ def run_bolts(command, parser, cl_args, unknown_args):
for i, (bolt, stat, header) in enumerate(bolts_result):
if i != 0:
print('')
- print('\'%s\' metrics:' % bolt)
- print(tabulate(stat, headers=header))
+ print(('\'%s\' metrics:' % bolt))
+ print((tabulate(stat, headers=header)))
return True
# pylint: disable=too-many-locals,superfluous-parens
@@ -174,11 +174,11 @@ def run_containers(command, parser, cl_args, unknown_args):
return False
containers = result['physical_plan']['stmgrs']
all_bolts, all_spouts = set(), set()
- for _, bolts in result['physical_plan']['bolts'].items():
+ for _, bolts in list(result['physical_plan']['bolts'].items()):
all_bolts = all_bolts | set(bolts)
- for _, spouts in result['physical_plan']['spouts'].items():
+ for _, spouts in list(result['physical_plan']['spouts'].items()):
all_spouts = all_spouts | set(spouts)
- stmgrs = containers.keys()
+ stmgrs = list(containers.keys())
stmgrs.sort()
if container_id is not None:
try:
@@ -201,5 +201,5 @@ def run_containers(command, parser, cl_args, unknown_args):
table.append([cid, host, port, pid, bolt_nums, spout_nums, len(instances)])
headers = ["container", "host", "port", "pid", "#bolt", "#spout", "#instance"]
sys.stdout.flush()
- print(tabulate(table, headers=headers))
+ print((tabulate(table, headers=headers)))
return True
diff --git a/heron/tools/explorer/src/python/topologies.py b/heron/tools/explorer/src/python/topologies.py
index 2328d7a..1f17d5e 100644
--- a/heron/tools/explorer/src/python/topologies.py
+++ b/heron/tools/explorer/src/python/topologies.py
@@ -44,8 +44,8 @@ def to_table(result):
''' normalize raw result to table '''
max_count = 20
table, count = [], 0
- for role, envs_topos in result.items():
- for env, topos in envs_topos.items():
+ for role, envs_topos in list(result.items()):
+ for env, topos in list(envs_topos.items()):
for topo in topos:
count += 1
if count > max_count:
@@ -70,10 +70,10 @@ def show_cluster(cl_args, cluster):
Log.error("Fail to connect to tracker: \'%s\'", cl_args["tracker_url"])
return False
table, header, rest_count = to_table(result)
- print('Topologies running in cluster \'%s\'' % cluster)
+ print(('Topologies running in cluster \'%s\'' % cluster))
if rest_count:
- print(' with %d more...' % rest_count)
- print(tabulate(table, headers=header))
+ print((' with %d more...' % rest_count))
+ print((tabulate(table, headers=header)))
return True
@@ -90,10 +90,10 @@ def show_cluster_role(cl_args, cluster, role):
Log.error("Fail to connect to tracker: \'%s\'", cl_args["tracker_url"])
return False
table, header, rest_count = to_table(result)
- print('Topologies running in cluster \'%s\' submitted by \'%s\':' % (cluster, role))
+ print(('Topologies running in cluster \'%s\' submitted by \'%s\':' % (cluster, role)))
if rest_count:
- print(' with %d more...' % rest_count)
- print(tabulate(table, headers=header))
+ print((' with %d more...' % rest_count))
+ print((tabulate(table, headers=header)))
return True
@@ -110,11 +110,11 @@ def show_cluster_role_env(cl_args, cluster, role, env):
Log.error("Fail to connect to tracker: \'%s\'", cl_args["tracker_url"])
return False
table, header, rest_count = to_table(result)
- print('Topologies running in cluster \'%s\', submitted by \'%s\', and\
- under environment \'%s\':' % (cluster, role, env))
+ print(('Topologies running in cluster \'%s\', submitted by \'%s\', and\
+ under environment \'%s\':' % (cluster, role, env)))
if rest_count:
- print(' with %d more...' % rest_count)
- print(tabulate(table, headers=header))
+ print((' with %d more...' % rest_count))
+ print((tabulate(table, headers=header)))
return True
# pylint: disable=unused-argument
diff --git a/heron/tools/tracker/src/python/config.py b/heron/tools/tracker/src/python/config.py
index 08887b8..089b03e 100644
--- a/heron/tools/tracker/src/python/config.py
+++ b/heron/tools/tracker/src/python/config.py
@@ -71,7 +71,7 @@ class Config(object):
"${USER}": "user",
}
dummy_formatted_url = url_format
- for key, value in valid_parameters.items():
+ for key, value in list(valid_parameters.items()):
dummy_formatted_url = dummy_formatted_url.replace(key, value)
# All $ signs must have been replaced
@@ -99,7 +99,7 @@ class Config(object):
formatted_url = formatter
- for key, value in common_parameters.items():
+ for key, value in list(common_parameters.items()):
formatted_url = formatted_url.replace(key, value)
return formatted_url
diff --git a/heron/tools/tracker/src/python/handlers/__init__.py b/heron/tools/tracker/src/python/handlers/__init__.py
index 497a01d..5305246 100644
--- a/heron/tools/tracker/src/python/handlers/__init__.py
+++ b/heron/tools/tracker/src/python/handlers/__init__.py
@@ -15,31 +15,31 @@
# specific language governing permissions and limitations
# under the License.
''' __init__ '''
-from basehandler import BaseHandler
-from clustershandler import ClustersHandler
-from containerfilehandler import ContainerFileDataHandler
-from containerfilehandler import ContainerFileDownloadHandler
-from containerfilehandler import ContainerFileStatsHandler
-from defaulthandler import DefaultHandler
-from exceptionhandler import ExceptionHandler
-from exceptionsummaryhandler import ExceptionSummaryHandler
-from executionstatehandler import ExecutionStateHandler
-from jmaphandler import JmapHandler
-from jstackhandler import JstackHandler
-from logicalplanhandler import LogicalPlanHandler
-from machineshandler import MachinesHandler
-from mainhandler import MainHandler
-from memoryhistogramhandler import MemoryHistogramHandler
-from metadatahandler import MetaDataHandler
-from metricshandler import MetricsHandler
-from metricsqueryhandler import MetricsQueryHandler
-from metricstimelinehandler import MetricsTimelineHandler
-from physicalplanhandler import PhysicalPlanHandler
-from packingplanhandler import PackingPlanHandler
-from pidhandler import PidHandler
-from runtimestatehandler import RuntimeStateHandler
-from schedulerlocationhandler import SchedulerLocationHandler
-from stateshandler import StatesHandler
-from topologieshandler import TopologiesHandler
-from topologyconfighandler import TopologyConfigHandler
-from topologyhandler import TopologyHandler
+from .basehandler import BaseHandler
+from .clustershandler import ClustersHandler
+from .containerfilehandler import ContainerFileDataHandler
+from .containerfilehandler import ContainerFileDownloadHandler
+from .containerfilehandler import ContainerFileStatsHandler
+from .defaulthandler import DefaultHandler
+from .exceptionhandler import ExceptionHandler
+from .exceptionsummaryhandler import ExceptionSummaryHandler
+from .executionstatehandler import ExecutionStateHandler
+from .jmaphandler import JmapHandler
+from .jstackhandler import JstackHandler
+from .logicalplanhandler import LogicalPlanHandler
+from .machineshandler import MachinesHandler
+from .mainhandler import MainHandler
+from .memoryhistogramhandler import MemoryHistogramHandler
+from .metadatahandler import MetaDataHandler
+from .metricshandler import MetricsHandler
+from .metricsqueryhandler import MetricsQueryHandler
+from .metricstimelinehandler import MetricsTimelineHandler
+from .physicalplanhandler import PhysicalPlanHandler
+from .packingplanhandler import PackingPlanHandler
+from .pidhandler import PidHandler
+from .runtimestatehandler import RuntimeStateHandler
+from .schedulerlocationhandler import SchedulerLocationHandler
+from .stateshandler import StatesHandler
+from .topologieshandler import TopologiesHandler
+from .topologyconfighandler import TopologyConfigHandler
+from .topologyhandler import TopologyHandler
diff --git a/heron/tools/tracker/src/python/handlers/logicalplanhandler.py b/heron/tools/tracker/src/python/handlers/logicalplanhandler.py
index 186cf70..2da1cd2 100644
--- a/heron/tools/tracker/src/python/handlers/logicalplanhandler.py
+++ b/heron/tools/tracker/src/python/handlers/logicalplanhandler.py
@@ -57,7 +57,7 @@ class LogicalPlanHandler(BaseHandler):
# format the logical plan as required by the web (because of Ambrose)
# first, spouts followed by bolts
spouts_map = dict()
- for name, value in lplan['spouts'].items():
+ for name, value in list(lplan['spouts'].items()):
spouts_map[name] = dict(
config=value.get("config", dict()),
outputs=value["outputs"],
@@ -67,7 +67,7 @@ class LogicalPlanHandler(BaseHandler):
)
bolts_map = dict()
- for name, value in lplan['bolts'].items():
+ for name, value in list(lplan['bolts'].items()):
bolts_map[name] = dict(
config=value.get("config", dict()),
inputComponents=[i['component_name'] for i in value['inputs']],
diff --git a/heron/tools/tracker/src/python/handlers/runtimestatehandler.py b/heron/tools/tracker/src/python/handlers/runtimestatehandler.py
index 6148605..b776517 100644
--- a/heron/tools/tracker/src/python/handlers/runtimestatehandler.py
+++ b/heron/tools/tracker/src/python/handlers/runtimestatehandler.py
@@ -116,7 +116,7 @@ class RuntimeStateHandler(BaseHandler):
topology = self.tracker.getTopologyByClusterRoleEnvironAndName(
cluster, role, environ, topology_name)
reg_summary = yield tornado.gen.Task(self.getStmgrsRegSummary, topology.tmaster)
- for stmgr, reg in reg_summary.items():
+ for stmgr, reg in list(reg_summary.items()):
runtime_state["stmgrs"].setdefault(stmgr, {})["is_registered"] = reg
self.write_success_response(runtime_state)
except Exception as e:
diff --git a/heron/tools/tracker/src/python/javaobj.py b/heron/tools/tracker/src/python/javaobj.py
index 65e9c9a..3f580ee 100644
--- a/heron/tools/tracker/src/python/javaobj.py
+++ b/heron/tools/tracker/src/python/javaobj.py
@@ -28,7 +28,7 @@ library marshal, pickle and json modules.
See: http://download.oracle.com/javase/6/docs/platform/serialization/spec/protocol.html
"""
-import StringIO
+import io
import struct
from heron.common.src.python.utils.log import Log
@@ -58,7 +58,7 @@ def loads(string):
Deserializes Java objects and primitive data serialized by ObjectOutputStream
from a string.
"""
- f = StringIO.StringIO(string)
+ f = io.StringIO(string)
marshaller = JavaObjectUnmarshaller(f)
marshaller.add_transformer(DefaultObjectTransformer())
return marshaller.readObject()
@@ -492,7 +492,7 @@ classDescFlags: 0x%X" % (serialVersionUID, newHandle, classDescFlags), ident)
def _create_hexdump(self, src, length=16):
FILTER = ''.join([(len(repr(chr(x))) == 3) and chr(x) or '.' for x in range(256)])
result = []
- for i in xrange(0, len(src), length):
+ for i in range(0, len(src), length):
s = src[i:i+length]
hexa = ' '.join(["%02X"%ord(x) for x in s])
printable = s.translate(FILTER)
@@ -558,7 +558,7 @@ class JavaObjectMarshaller(JavaObjectConstants):
# pylint: disable=attribute-defined-outside-init
def dump(self, obj):
self.object_obj = obj
- self.object_stream = StringIO.StringIO()
+ self.object_stream = io.StringIO()
self._writeStreamHeader()
self.writeObject(obj)
return self.object_stream.getvalue()
diff --git a/heron/tools/tracker/src/python/main.py b/heron/tools/tracker/src/python/main.py
index d9f60e9..1949e0a 100644
--- a/heron/tools/tracker/src/python/main.py
+++ b/heron/tools/tracker/src/python/main.py
@@ -19,7 +19,7 @@
# under the License.
''' main.py '''
-from __future__ import print_function
+
import argparse
import os
import signal
@@ -109,7 +109,7 @@ class _HelpAction(argparse._HelpAction):
# but better save than sorry
for subparsers_action in subparsers_actions:
# get all subparsers and print help
- for choice, subparser in subparsers_action.choices.items():
+ for choice, subparser in list(subparsers_action.choices.items()):
print("Subparser '{}'".format(choice))
print(subparser.format_help())
diff --git a/heron/tools/tracker/src/python/query_operators.py b/heron/tools/tracker/src/python/query_operators.py
index d9641cb..dd348a7 100644
--- a/heron/tools/tracker/src/python/query_operators.py
+++ b/heron/tools/tracker/src/python/query_operators.py
@@ -54,7 +54,7 @@ class Metrics(object):
def floorTimestamps(self, start, end, timeline):
""" floor timestamp """
ret = {}
- for timestamp, value in timeline.items():
+ for timestamp, value in list(timeline.items()):
ts = timestamp / 60 * 60
if start <= ts <= end:
ret[ts] = value
@@ -143,9 +143,9 @@ class TS(Operator):
}
timelines = metrics["timeline"][self.metricName]
allMetrics = []
- for instance, timeline in timelines.items():
+ for instance, timeline in list(timelines.items()):
toBeDeletedKeys = []
- for key, value in timeline.items():
+ for key, value in list(timeline.items()):
floatValue = float(value)
# Check if the value is really float or not.
# In python, float("nan") returns "nan" which is actually a float value,
@@ -207,9 +207,9 @@ class Sum(Operator):
def execute(self, tracker, tmaster, start, end):
# Initialize the metric to be returned with sum of all the constants.
retMetrics = Metrics(None, None, None, start, end, {})
- constants = filter(lambda ts: isinstance(ts, float), self.timeSeriesList)
+ constants = [ts for ts in self.timeSeriesList if isinstance(ts, float)]
retMetrics.setDefault(sum(constants), start, end)
- leftOverTimeSeries = filter(lambda ts: not isinstance(ts, float), self.timeSeriesList)
+ leftOverTimeSeries = [ts for ts in self.timeSeriesList if not isinstance(ts, float)]
futureMetrics = []
for timeseries in leftOverTimeSeries:
@@ -225,7 +225,7 @@ class Sum(Operator):
# Aggregate all of the them
for metric in allMetrics:
- for timestamp, value in metric.timeline.items():
+ for timestamp, value in list(metric.timeline.items()):
if timestamp in retMetrics.timeline:
retMetrics.timeline[timestamp] += value
raise tornado.gen.Return([retMetrics])
@@ -249,10 +249,10 @@ class Max(Operator):
def execute(self, tracker, tmaster, start, end):
# Initialize the metric to be returned with max of all the constants.
retMetrics = Metrics(None, None, None, start, end, {})
- constants = filter(lambda ts: isinstance(ts, float), self.timeSeriesList)
+ constants = [ts for ts in self.timeSeriesList if isinstance(ts, float)]
if constants:
retMetrics.setDefault(max(constants), start, end)
- leftOverTimeSeries = filter(lambda ts: not isinstance(ts, float), self.timeSeriesList)
+ leftOverTimeSeries = [ts for ts in self.timeSeriesList if not isinstance(ts, float)]
futureMetrics = []
for timeseries in leftOverTimeSeries:
@@ -269,7 +269,7 @@ class Max(Operator):
# Aggregate all of the them
for metric in allMetrics:
- for timestamp, value in metric.timeline.items():
+ for timestamp, value in list(metric.timeline.items()):
if start <= timestamp <= end:
if timestamp not in retMetrics.timeline:
retMetrics.timeline[timestamp] = value
@@ -302,7 +302,7 @@ class Percentile(Operator):
@tornado.gen.coroutine
def execute(self, tracker, tmaster, start, end):
- leftOverTimeSeries = filter(lambda ts: not isinstance(ts, float), self.timeSeriesList)
+ leftOverTimeSeries = [ts for ts in self.timeSeriesList if not isinstance(ts, float)]
futureMetrics = []
for timeseries in leftOverTimeSeries:
@@ -323,14 +323,14 @@ class Percentile(Operator):
# Aggregate all of the them
for metric in allMetrics:
- for timestamp, value in metric.timeline.items():
+ for timestamp, value in list(metric.timeline.items()):
if start <= timestamp <= end:
if timestamp not in timeline:
timeline[timestamp] = []
timeline[timestamp].append(value)
retTimeline = {}
- for timestamp, values in timeline.items():
+ for timestamp, values in list(timeline.items()):
if not values:
continue
index = int(self.quantile * 1.0 * (len(values) - 1) / 100.0)
@@ -429,7 +429,7 @@ class Divide(Operator):
if key not in metrics2:
continue
met = Metrics(None, None, key, start, end, {})
- for timestamp in metrics[key].timeline.keys():
+ for timestamp in list(metrics[key].timeline.keys()):
if timestamp not in metrics2[key].timeline or metrics2[key].timeline[timestamp] == 0:
metrics[key].timeline.pop(timestamp)
else:
@@ -440,11 +440,11 @@ class Divide(Operator):
# If first is univariate
elif len(metrics) == 1 and "" in metrics:
allMetrics = []
- for key, metric in metrics2.items():
+ for key, metric in list(metrics2.items()):
# Initialize with first metrics timeline, but second metric's instance
# because that is multivariate
met = Metrics(None, None, metric.instance, start, end, dict(metrics[""].timeline))
- for timestamp in met.timeline.keys():
+ for timestamp in list(met.timeline.keys()):
if timestamp not in metric.timeline or metric.timeline[timestamp] == 0:
met.timeline.pop(timestamp)
else:
@@ -454,10 +454,10 @@ class Divide(Operator):
# If second is univariate
else:
allMetrics = []
- for key, metric in metrics.items():
+ for key, metric in list(metrics.items()):
# Initialize with first metrics timeline and its instance
met = Metrics(None, None, metric.instance, start, end, dict(metric.timeline))
- for timestamp in met.timeline.keys():
+ for timestamp in list(met.timeline.keys()):
if timestamp not in metrics2[""].timeline or metrics2[""].timeline[timestamp] == 0:
met.timeline.pop(timestamp)
else:
@@ -554,7 +554,7 @@ class Multiply(Operator):
if key not in metrics2:
continue
met = Metrics(None, None, key, start, end, {})
- for timestamp in metrics[key].timeline.keys():
+ for timestamp in list(metrics[key].timeline.keys()):
if timestamp not in metrics2[key].timeline:
metrics[key].timeline.pop(timestamp)
else:
@@ -565,11 +565,11 @@ class Multiply(Operator):
# If first is univariate
elif len(metrics) == 1 and "" in metrics:
allMetrics = []
- for key, metric in metrics2.items():
+ for key, metric in list(metrics2.items()):
# Initialize with first metrics timeline, but second metric's instance
# because that is multivariate
met = Metrics(None, None, metric.instance, start, end, dict(metrics[""].timeline))
- for timestamp in met.timeline.keys():
+ for timestamp in list(met.timeline.keys()):
if timestamp not in metric.timeline:
met.timeline.pop(timestamp)
else:
@@ -579,10 +579,10 @@ class Multiply(Operator):
# If second is univariate
else:
allMetrics = []
- for key, metric in metrics.items():
+ for key, metric in list(metrics.items()):
# Initialize with first metrics timeline and its instance
met = Metrics(None, None, metric.instance, start, end, dict(metric.timeline))
- for timestamp in met.timeline.keys():
+ for timestamp in list(met.timeline.keys()):
if timestamp not in metrics2[""].timeline:
met.timeline.pop(timestamp)
else:
@@ -677,7 +677,7 @@ class Subtract(Operator):
if key not in metrics2:
continue
met = Metrics(None, None, key, start, end, {})
- for timestamp in metrics[key].timeline.keys():
+ for timestamp in list(metrics[key].timeline.keys()):
if timestamp not in metrics2[key].timeline:
metrics[key].timeline.pop(timestamp)
else:
@@ -688,11 +688,11 @@ class Subtract(Operator):
# If first is univariate
elif len(metrics) == 1 and "" in metrics:
allMetrics = []
- for key, metric in metrics2.items():
+ for key, metric in list(metrics2.items()):
# Initialize with first metrics timeline, but second metric's instance
# because that is multivariate
met = Metrics(None, None, metric.instance, start, end, dict(metrics[""].timeline))
- for timestamp in met.timeline.keys():
+ for timestamp in list(met.timeline.keys()):
if timestamp not in metric.timeline:
met.timeline.pop(timestamp)
else:
@@ -702,10 +702,10 @@ class Subtract(Operator):
# If second is univariate
else:
allMetrics = []
- for key, metric in metrics.items():
+ for key, metric in list(metrics.items()):
# Initialize with first metrics timeline and its instance
met = Metrics(None, None, metric.instance, start, end, dict(metric.timeline))
- for timestamp in met.timeline.keys():
+ for timestamp in list(met.timeline.keys()):
if timestamp not in metrics2[""].timeline:
met.timeline.pop(timestamp)
else:
diff --git a/heron/tools/tracker/src/python/topology.py b/heron/tools/tracker/src/python/topology.py
index 5198492..2deef9e 100644
--- a/heron/tools/tracker/src/python/topology.py
+++ b/heron/tools/tracker/src/python/topology.py
@@ -108,7 +108,7 @@ class Topology(object):
unregister the corresponding watch.
"""
to_remove = []
- for uid, callback in self.watches.items():
+ for uid, callback in list(self.watches.items()):
try:
callback(self)
except Exception as e:
@@ -202,7 +202,7 @@ class Topology(object):
"""
Returns a list of names of all the spouts
"""
- return map(lambda component: component.comp.name, self.spouts())
+ return [component.comp.name for component in self.spouts()]
def bolts(self):
"""
@@ -216,7 +216,7 @@ class Topology(object):
"""
Returns a list of names of all the bolts
"""
- return map(lambda component: component.comp.name, self.bolts())
+ return [component.comp.name for component in self.bolts()]
def get_machines(self):
"""
@@ -225,7 +225,7 @@ class Topology(object):
"""
if self.physical_plan:
stmgrs = list(self.physical_plan.stmgrs)
- return map(lambda s: s.host_name, stmgrs)
+ return [s.host_name for s in stmgrs]
return []
def get_status(self):
diff --git a/heron/tools/tracker/src/python/tracker.py b/heron/tools/tracker/src/python/tracker.py
index 15df468..338e02c 100644
--- a/heron/tools/tracker/src/python/tracker.py
+++ b/heron/tools/tracker/src/python/tracker.py
@@ -135,7 +135,7 @@ class Tracker(object):
Log.info("State watch triggered for topologies.")
Log.debug("Topologies: " + str(topologies))
existingTopologies = self.getTopologiesForStateLocation(state_manager.name)
- existingTopNames = map(lambda t: t.name, existingTopologies)
+ existingTopNames = [t.name for t in existingTopologies]
Log.debug("Existing topologies: " + str(existingTopNames))
for name in existingTopNames:
if name not in topologies:
@@ -164,10 +164,10 @@ class Tracker(object):
an optional role.
Raises exception if topology is not found, or more than one are found.
"""
- topologies = list(filter(lambda t: t.name == topologyName
+ topologies = list([t for t in self.topologies if t.name == topologyName
and t.cluster == cluster
and (not role or t.execution_state.role == role)
- and t.environ == environ, self.topologies))
+ and t.environ == environ])
if not topologies or len(topologies) > 1:
if role is not None:
raise Exception("Topology not found for {0}, {1}, {2}, {3}".format(
@@ -183,7 +183,7 @@ class Tracker(object):
"""
Returns all the topologies for a given state manager.
"""
- return filter(lambda t: t.state_manager_name == name, self.topologies)
+ return [t for t in self.topologies if t.state_manager_name == name]
def addNewTopology(self, state_manager, topologyName):
"""
@@ -672,7 +672,7 @@ class Tracker(object):
Raises exception if no such topology is found.
"""
# Iterate over the values to filter the desired topology.
- for (topology_name, _), topologyInfo in self.topologyInfos.items():
+ for (topology_name, _), topologyInfo in list(self.topologyInfos.items()):
executionState = topologyInfo["execution_state"]
if (topologyName == topology_name and
cluster == executionState["cluster"] and
diff --git a/heron/tools/tracker/tests/python/tracker_unittest.py b/heron/tools/tracker/tests/python/tracker_unittest.py
index 136f5d6..fcb4810 100644
--- a/heron/tools/tracker/tests/python/tracker_unittest.py
+++ b/heron/tools/tracker/tests/python/tracker_unittest.py
@@ -206,12 +206,12 @@ class TrackerTest(unittest.TestCase):
self.tracker.addNewTopology(mock_state_manager_1, 'top_name1')
self.assertItemsEqual(
['top_name1'],
- map(lambda t: t.name, self.tracker.topologies))
+ [t.name for t in self.tracker.topologies])
self.tracker.addNewTopology(mock_state_manager_1, 'top_name2')
self.assertItemsEqual(
['top_name1', 'top_name2'],
- map(lambda t: t.name, self.tracker.topologies))
+ [t.name for t in self.tracker.topologies])
self.assertEqual(2, mock_state_manager_1.get_pplan.call_count)
self.assertEqual(2, mock_state_manager_1.get_execution_state.call_count)
@@ -261,11 +261,11 @@ class TrackerTest(unittest.TestCase):
self.assertEqual(packing_plan['id'], 'ExclamationTopology')
self.assertEqual(packing_plan['container_plans'][0]['id'], 1)
self.assertEqual(packing_plan['container_plans'][0]['required_resources'],
- {'disk': 2048L, 'ram': 1024L, 'cpu': 1.0})
+ {'disk': 2048, 'ram': 1024, 'cpu': 1.0})
self.assertEqual(packing_plan['container_plans'][0]['instances'][0],
{
'component_index': 1,
- 'component_name': u'word',
- 'instance_resources': {'cpu': 1.0, 'disk': 2048L, 'ram': 1024L},
+ 'component_name': 'word',
+ 'instance_resources': {'cpu': 1.0, 'disk': 2048, 'ram': 1024},
'task_id': 1
})
diff --git a/heron/tools/ui/src/python/args.py b/heron/tools/ui/src/python/args.py
index b4bcffb..73384a7 100644
--- a/heron/tools/ui/src/python/args.py
+++ b/heron/tools/ui/src/python/args.py
@@ -39,9 +39,9 @@ class _HelpAction(argparse._HelpAction):
# but better save than sorry
for subparsers_action in subparsers_actions:
# get all subparsers and print help
- for choice, subparser in subparsers_action.choices.items():
- print("Subparser '{}'".format(choice))
- print(subparser.format_help())
+ for choice, subparser in list(subparsers_action.choices.items()):
+ print(("Subparser '{}'".format(choice)))
+ print((subparser.format_help()))
parser.exit()
diff --git a/heron/tools/ui/src/python/handlers/api/topology.py b/heron/tools/ui/src/python/handlers/api/topology.py
index 3df74d5..f7c9100 100644
--- a/heron/tools/ui/src/python/handlers/api/topology.py
+++ b/heron/tools/ui/src/python/handlers/api/topology.py
@@ -53,8 +53,8 @@ class TopologyExceptionSummaryHandler(base.BaseHandler):
if not 'spouts' in lplan or not 'bolts' in lplan:
self.write(dict())
return
- comp_names = lplan['spouts'].keys()
- comp_names.extend(lplan['bolts'].keys())
+ comp_names = list(lplan['spouts'].keys())
+ comp_names.extend(list(lplan['bolts'].keys()))
else:
comp_names = [comp_name]
exception_infos = dict()
@@ -64,7 +64,7 @@ class TopologyExceptionSummaryHandler(base.BaseHandler):
# Combine exceptions from multiple component
aggregate_exceptions = dict()
- for comp_name, exception_logs in exception_infos.items():
+ for comp_name, exception_logs in list(exception_infos.items()):
for exception_log in exception_logs:
class_name = exception_log['class_name']
if class_name != '':
@@ -96,11 +96,11 @@ class ListTopologiesJsonHandler(base.BaseHandler):
result = dict()
# now convert some of the fields to be displayable
- for cluster, cluster_value in topologies.items():
+ for cluster, cluster_value in list(topologies.items()):
result[cluster] = dict()
- for environ, environ_value in cluster_value.items():
+ for environ, environ_value in list(cluster_value.items()):
result[cluster][environ] = dict()
- for topology, topology_value in environ_value.items():
+ for topology, topology_value in list(environ_value.items()):
if "jobname" not in topology_value or topology_value["jobname"] is None:
continue
diff --git a/heron/tools/ui/src/python/handlers/mainhandler.py b/heron/tools/ui/src/python/handlers/mainhandler.py
index fe5b7a3..a3d82f2 100644
--- a/heron/tools/ui/src/python/handlers/mainhandler.py
+++ b/heron/tools/ui/src/python/handlers/mainhandler.py
@@ -29,4 +29,4 @@ class MainHandler(base.BaseHandler):
'''
:return:
'''
- self.redirect(u"/topologies")
+ self.redirect("/topologies")
diff --git a/heron/tools/ui/src/python/handlers/ranges.py b/heron/tools/ui/src/python/handlers/ranges.py
index 20ddc82..be1f5ec 100644
--- a/heron/tools/ui/src/python/handlers/ranges.py
+++ b/heron/tools/ui/src/python/handlers/ranges.py
@@ -49,7 +49,7 @@ def get_time_ranges(ranges):
# form the new
time_slots = dict()
- for key, value in ranges.items():
+ for key, value in list(ranges.items()):
time_slots[key] = (now - value[0], now - value[1], value[2])
return (now, time_slots)
diff --git a/heron/tools/ui/src/python/handlers/topology.py b/heron/tools/ui/src/python/handlers/topology.py
index b08e13d..51985fe 100644
--- a/heron/tools/ui/src/python/handlers/topology.py
+++ b/heron/tools/ui/src/python/handlers/topology.py
@@ -25,8 +25,8 @@ import tornado.escape
import tornado.web
import tornado.gen
-import base
-import common
+from . import base
+from . import common
import heron.tools.common.src.python.access as access
import heron.common.src.python.utils.log as log
diff --git a/heron/tools/ui/src/python/main.py b/heron/tools/ui/src/python/main.py
index cb8b4b7..07ca43c 100644
--- a/heron/tools/ui/src/python/main.py
+++ b/heron/tools/ui/src/python/main.py
@@ -19,7 +19,7 @@
# under the License.
''' main.py '''
-from __future__ import print_function
+
import logging
import os
import signal
diff --git a/heronpy/api/cloudpickle.py b/heronpy/api/cloudpickle.py
index f49b7e1..9d18ae7 100644
--- a/heronpy/api/cloudpickle.py
+++ b/heronpy/api/cloudpickle.py
@@ -34,7 +34,7 @@ LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
"""
-from __future__ import print_function
+
import operator
import opcode
@@ -55,12 +55,12 @@ import weakref
if sys.version < '3':
from pickle import Pickler # pylint: disable=ungrouped-imports
try:
- from cStringIO import StringIO
+ from io import StringIO
except ImportError:
- from StringIO import StringIO
+ from io import StringIO
PY3 = False
else:
- types.ClassType = type
+ type = type
from pickle import _Pickler as Pickler # pylint: disable=ungrouped-imports
from io import BytesIO as StringIO # pylint: disable=ungrouped-imports
PY3 = True
@@ -79,7 +79,7 @@ def islambda(func):
_BUILTIN_TYPE_NAMES = {}
-for k1, v1 in types.__dict__.items():
+for k1, v1 in list(types.__dict__.items()):
if type(v1) is type: # pylint: disable=unidiomatic-typecheck
_BUILTIN_TYPE_NAMES[v1] = k1
@@ -96,7 +96,7 @@ if sys.version_info < (3, 4):
"""
code = getattr(code, 'co_code', b'')
if not PY3:
- code = map(ord, code)
+ code = list(map(ord, code))
n = len(code)
i = 0
@@ -169,7 +169,7 @@ class CloudPickler(Pickler): # pylint: disable=too-many-public-methods
dispatch[types.GeneratorType] = save_unsupported
# itertools objects do not pickle!
- for v in itertools.__dict__.values():
+ for v in list(itertools.__dict__.values()):
if type(v) is type: # pylint: disable=unidiomatic-typecheck
dispatch[v] = save_unsupported
@@ -386,7 +386,7 @@ class CloudPickler(Pickler): # pylint: disable=too-many-public-methods
return Pickler.save_global(self, obj, name)
typ = type(obj)
- if typ is not obj and isinstance(obj, (type, types.ClassType)):
+ if typ is not obj and isinstance(obj, type):
d = dict(obj.__dict__) # copy dict proxy to a dict
if not isinstance(d.get('__dict__', None), property):
# don't extract dict that are properties
@@ -408,7 +408,7 @@ class CloudPickler(Pickler): # pylint: disable=too-many-public-methods
d.pop('__doc__', None)
# handle property and staticmethod
dd = {}
- for k, v in d.items():
+ for k, v in list(d.items()):
if isinstance(v, property):
k = ('property', k)
v = (v.fget, v.fset, v.fdel, v.__doc__)
@@ -427,7 +427,7 @@ class CloudPickler(Pickler): # pylint: disable=too-many-public-methods
raise pickle.PicklingError("Can't pickle %r" % obj)
dispatch[type] = save_global
- dispatch[types.ClassType] = save_global
+ dispatch[type] = save_global
def save_instancemethod(self, obj):
# Memoization rarely is ever useful due to python bounding
@@ -604,7 +604,7 @@ class CloudPickler(Pickler): # pylint: disable=too-many-public-methods
def save_file(self, obj): # pylint: disable=too-many-branches
"""Save a file"""
try:
- import StringIO as pystringIO #we can't use cStringIO as it lacks the name attribute
+ import io as pystringIO #we can't use cStringIO as it lacks the name attribute
except ImportError:
import io as pystringIO # pylint: disable=reimported
@@ -708,7 +708,7 @@ def subimport(name):
# restores function attributes
def _restore_attr(obj, attr):
- for key, val in attr.items():
+ for key, val in list(attr.items()):
setattr(obj, key, val)
return obj
@@ -789,7 +789,7 @@ def _load_class(cls, d):
"""
Loads additional properties into class `cls`.
"""
- for k, v in d.items():
+ for k, v in list(d.items()):
if isinstance(k, tuple):
typ, k = k
if typ == 'property':
diff --git a/heronpy/api/component/component_spec.py b/heronpy/api/component/component_spec.py
index 7e23772..bf134d7 100644
--- a/heronpy/api/component/component_spec.py
+++ b/heronpy/api/component/component_spec.py
@@ -115,7 +115,7 @@ class HeronComponentSpec(object):
# iterate through self.custom_config
if self.custom_config is not None:
sanitized = self._sanitize_config(self.custom_config)
- for key, value in sanitized.items():
+ for key, value in list(sanitized.items()):
if isinstance(value, str):
kvs = proto_config.kvs.add()
kvs.key = key
@@ -149,7 +149,7 @@ class HeronComponentSpec(object):
raise TypeError("Component-specific configuration must be given as a dict type, given: %s"
% str(type(custom_config)))
sanitized = {}
- for key, value in custom_config.items():
+ for key, value in list(custom_config.items()):
if not isinstance(key, str):
raise TypeError("Key for component-specific configuration must be string, given: %s:%s"
% (str(type(key)), str(key)))
@@ -170,7 +170,7 @@ class HeronComponentSpec(object):
# sanitize inputs and get a map <GlobalStreamId -> Grouping>
input_dict = self._sanitize_inputs()
- for global_streamid, gtype in input_dict.items():
+ for global_streamid, gtype in list(input_dict.items()):
in_stream = bolt.inputs.add()
in_stream.stream.CopyFrom(self._get_stream_id(global_streamid.component_id,
global_streamid.stream_id))
@@ -196,7 +196,7 @@ class HeronComponentSpec(object):
if isinstance(self.inputs, dict):
# inputs are dictionary, must be either <HeronComponentSpec -> Grouping> or
# <GlobalStreamId -> Grouping>
- for key, grouping in self.inputs.items():
+ for key, grouping in list(self.inputs.items()):
if not Grouping.is_grouping_sane(grouping):
raise ValueError('A given grouping is not supported')
if isinstance(key, HeronComponentSpec):
@@ -239,7 +239,7 @@ class HeronComponentSpec(object):
# sanitize outputs and get a map <stream_id -> out fields>
output_map = self._sanitize_outputs()
- for stream_id, out_fields in output_map.items():
+ for stream_id, out_fields in list(output_map.items()):
out_stream = spbl.outputs.add()
out_stream.stream.CopyFrom(self._get_stream_id(self.name, stream_id))
out_stream.schema.CopyFrom(self._get_stream_schema(out_fields))
diff --git a/heronpy/api/metrics.py b/heronpy/api/metrics.py
index 89f31fa..b944d58 100644
--- a/heronpy/api/metrics.py
+++ b/heronpy/api/metrics.py
@@ -63,7 +63,7 @@ class MultiCountMetric(IMetric):
def get_value_and_reset(self):
ret = {}
- for key, value in self.value.items():
+ for key, value in list(self.value.items()):
ret[key] = value.get_value_and_reset()
return ret
@@ -144,7 +144,7 @@ class MultiReducedMetric(IMetric):
def get_value_and_reset(self):
ret = {}
- for key, value in self.value.items():
+ for key, value in list(self.value.items()):
ret[key] = value.get_value_and_reset()
self.value[key] = value
return ret
diff --git a/heronpy/api/serializer.py b/heronpy/api/serializer.py
index 6a922f6..08174a2 100644
--- a/heronpy/api/serializer.py
+++ b/heronpy/api/serializer.py
@@ -22,7 +22,7 @@
from abc import abstractmethod
try:
- import cPickle as pickle
+ import pickle as pickle
except:
import pickle
diff --git a/heronpy/api/tests/python/component_unittest.py b/heronpy/api/tests/python/component_unittest.py
index a72adb3..9da4a8d 100644
--- a/heronpy/api/tests/python/component_unittest.py
+++ b/heronpy/api/tests/python/component_unittest.py
@@ -167,7 +167,7 @@ class ComponentSpecTest(unittest.TestCase):
inputs_list = [GlobalStreamId("spout1", "default"), GlobalStreamId("spout2", "some_stream")]
spec = HeronComponentSpec("bolt", "bl_cls", False, 1, inputs=inputs_list)
ret = spec._sanitize_inputs()
- self.assertEqual(ret, dict(zip(inputs_list, [Grouping.SHUFFLE] * 2)))
+ self.assertEqual(ret, dict(list(zip(inputs_list, [Grouping.SHUFFLE] * 2))))
# list of neither GlobalStreamId nor HeronComponentSpec
inputs_list = [None, 123, "string", [GlobalStreamId("sp", "default")]]
diff --git a/heronpy/api/tests/python/metrics_unittest.py b/heronpy/api/tests/python/metrics_unittest.py
index e7d7866..063651a 100644
--- a/heronpy/api/tests/python/metrics_unittest.py
+++ b/heronpy/api/tests/python/metrics_unittest.py
@@ -44,8 +44,8 @@ class MetricsTest(unittest.TestCase):
for _ in range(10):
for key in key_list:
metric.incr(key=key)
- self.assertEqual(metric.get_value_and_reset(), dict(zip(key_list, [10] * 3)))
- self.assertEqual(metric.get_value_and_reset(), dict(zip(key_list, [0] * 3)))
+ self.assertEqual(metric.get_value_and_reset(), dict(list(zip(key_list, [10] * 3))))
+ self.assertEqual(metric.get_value_and_reset(), dict(list(zip(key_list, [0] * 3))))
metric.add_key("key4")
ret = metric.get_value_and_reset()
@@ -71,8 +71,8 @@ class MetricsTest(unittest.TestCase):
metric.update(key=key_list[0], value=i)
metric.update(key=key_list[1], value=i * 2)
metric.update(key=key_list[2], value=i * 3)
- self.assertEqual(metric.get_value_and_reset(), dict(zip(key_list, [5.5, 11, 16.5])))
- self.assertEqual(metric.get_value_and_reset(), dict(zip(key_list, [None] * 3)))
+ self.assertEqual(metric.get_value_and_reset(), dict(list(zip(key_list, [5.5, 11, 16.5]))))
+ self.assertEqual(metric.get_value_and_reset(), dict(list(zip(key_list, [None] * 3))))
metric.add_key("key4")
ret = metric.get_value_and_reset()
diff --git a/heronpy/api/topology.py b/heronpy/api/topology.py
index ae7396f..c62a6bd 100644
--- a/heronpy/api/topology.py
+++ b/heronpy/api/topology.py
@@ -47,7 +47,7 @@ class TopologyType(type):
spout_specs = {}
# Copy HeronComponentSpec items out of class_dict
specs = TopologyType.class_dict_to_specs(class_dict)
- for spec in iter(specs.values()):
+ for spec in iter(list(specs.values())):
if spec.is_spout:
TopologyType.add_spout_specs(spec, spout_specs)
else:
@@ -73,7 +73,7 @@ class TopologyType(type):
"""Takes a class `__dict__` and returns `HeronComponentSpec` entries"""
specs = {}
- for name, spec in class_dict.items():
+ for name, spec in list(class_dict.items()):
if isinstance(spec, HeronComponentSpec):
# Use the variable name as the specification name.
if spec.name is None:
@@ -103,7 +103,7 @@ class TopologyType(type):
# add defaults
topo_config.update(mcs.DEFAULT_TOPOLOGY_CONFIG)
- for name, custom_config in class_dict.items():
+ for name, custom_config in list(class_dict.items()):
if name == 'config' and isinstance(custom_config, dict):
sanitized_dict = mcs._sanitize_config(custom_config)
topo_config.update(sanitized_dict)
@@ -131,7 +131,7 @@ class TopologyType(type):
config = topology_pb2.Config()
conf_dict = class_dict['_topo_config']
- for key, value in conf_dict.items():
+ for key, value in list(conf_dict.items()):
if isinstance(value, str):
kvs = config.kvs.add()
kvs.key = key
@@ -243,7 +243,7 @@ class TopologyType(type):
These values will need to be serialized before adding to a protobuf message.
"""
sanitized = {}
- for key, value in custom_config.items():
+ for key, value in list(custom_config.items()):
if not isinstance(key, str):
raise TypeError("Key for topology-wide configuration must be string, given: %s: %s"
% (str(type(key)), str(key)))
diff --git a/heronpy/streamlet/impl/contextimpl.py b/heronpy/streamlet/impl/contextimpl.py
index d13cce9..d944ec7 100644
--- a/heronpy/streamlet/impl/contextimpl.py
+++ b/heronpy/streamlet/impl/contextimpl.py
@@ -36,7 +36,7 @@ class ContextImpl(Context):
return self._top_context.get_cluster_config()
def get_stream_name(self):
- return self._top_context.get_this_sources().keys()[0].id
+ return list(self._top_context.get_this_sources().keys())[0].id
def get_num_partitions(self):
return len(self._top_context.get_component_tasks(self._top_context.get_component_id()))
diff --git a/heronpy/streamlet/impl/joinbolt.py b/heronpy/streamlet/impl/joinbolt.py
index efa64f7..ea76532 100644
--- a/heronpy/streamlet/impl/joinbolt.py
+++ b/heronpy/streamlet/impl/joinbolt.py
@@ -80,7 +80,7 @@ class JoinBolt(SlidingWindowBolt, StreamletBoltBase):
if not isinstance(userdata, collections.Iterable) or len(userdata) != 2:
raise RuntimeError("Join tuples must be iterable of length 2")
self._add(userdata[0], userdata[1], tup.component, mymap)
- for (key, values) in mymap.items():
+ for (key, values) in list(mymap.items()):
if self._join_type == JoinBolt.INNER:
if values[0] and values[1]:
self.inner_join_and_emit(key, values, window_config)
@@ -172,8 +172,8 @@ class JoinStreamlet(Streamlet):
# pylint: disable=superfluous-parens
def _build_this(self, builder, stage_names):
- print("join_build_this left: %s right: %s" % (self._left._built, self._right._built))
- print("left: %s right: %s" % (self._left.get_name(), self._right.get_name()))
+ print(("join_build_this left: %s right: %s" % (self._left._built, self._right._built)))
+ print(("left: %s right: %s" % (self._left.get_name(), self._right.get_name())))
if not self._left._built or not self._right._built:
return False
if not self.get_name():
diff --git a/heronpy/streamlet/impl/reducebykeyandwindowbolt.py b/heronpy/streamlet/impl/reducebykeyandwindowbolt.py
index a8267b9..7cd4a93 100644
--- a/heronpy/streamlet/impl/reducebykeyandwindowbolt.py
+++ b/heronpy/streamlet/impl/reducebykeyandwindowbolt.py
@@ -62,7 +62,7 @@ class ReduceByKeyAndWindowBolt(SlidingWindowBolt, StreamletBoltBase):
if not isinstance(userdata, collections.Iterable) or len(userdata) != 2:
raise RuntimeError("ReduceByWindow tuples must be iterable of length 2")
self._add(userdata[0], userdata[1], mymap)
- for (key, values) in mymap.items():
+ for (key, values) in list(mymap.items()):
result = values[0]
for value in values[1:]:
result = self.reduce_function(result, value)
diff --git a/integration_test/src/python/http_server/main.py b/integration_test/src/python/http_server/main.py
index 9a7dcd1..95c396c 100644
--- a/integration_test/src/python/http_server/main.py
+++ b/integration_test/src/python/http_server/main.py
@@ -108,13 +108,13 @@ class StateResultHandler(tornado.web.RequestHandler):
if key in self.result_map:
# fix the duplicate record issue
for comp in self.result_map[key]:
- if comp.keys()[0] == data.keys()[0]:
+ if list(comp.keys())[0] == list(data.keys())[0]:
break
else:
self.result_map[key].append(data)
else:
self.result_map[key] = [data]
- self.write("Results written successfully: topology " + key + ' instance ' + data.keys()[0])
+ self.write("Results written successfully: topology " + key + ' instance ' + list(data.keys())[0])
else:
raise tornado.web.HTTPError(status_code=404, log_message="Invalid key %s" % key)
diff --git a/integration_test/src/python/integration_test/core/aggregator_bolt.py b/integration_test/src/python/integration_test/core/aggregator_bolt.py
index 2460a42..f6acd4f 100644
--- a/integration_test/src/python/integration_test/core/aggregator_bolt.py
+++ b/integration_test/src/python/integration_test/core/aggregator_bolt.py
@@ -19,9 +19,9 @@
# under the License.
'''aggregator_bolt.py'''
-import httplib
+import http.client
import json
-from urlparse import urlparse
+from urllib.parse import urlparse
from heron.common.src.python.utils.log import Log
from ..core import constants as integ_constants
@@ -43,7 +43,7 @@ class AggregatorBolt(TerminalBolt):
self.result.append(tup.values[0])
def _post_result_to_server(self, json_result):
- conn = httplib.HTTPConnection(self.parsed_url.netloc)
+ conn = http.client.HTTPConnection(self.parsed_url.netloc)
conn.request("POST", self.parsed_url.path, json_result)
response = conn.getresponse()
if response.status == 200:
diff --git a/integration_test/src/python/integration_test/core/integration_test_bolt.py b/integration_test/src/python/integration_test/core/integration_test_bolt.py
index 41b0581..cfb4d7d 100644
--- a/integration_test/src/python/integration_test/core/integration_test_bolt.py
+++ b/integration_test/src/python/integration_test/core/integration_test_bolt.py
@@ -59,7 +59,7 @@ class IntegrationTestBolt(Bolt):
upstream_components = set()
self.terminal_to_receive = 0
- for streamId in context.get_this_sources().keys():
+ for streamId in list(context.get_this_sources().keys()):
# streamId is topology_pb2.StreamId protobuf message
upstream_components.add(streamId.component_name)
for comp_name in upstream_components:
diff --git a/integration_test/src/python/integration_test/core/test_topology_builder.py b/integration_test/src/python/integration_test/core/test_topology_builder.py
index e2a77a7..d64a874 100644
--- a/integration_test/src/python/integration_test/core/test_topology_builder.py
+++ b/integration_test/src/python/integration_test/core/test_topology_builder.py
@@ -122,7 +122,7 @@ class TestTopologyBuilder(TopologyBuilder):
# building a graph directed from children to parents, by looking only on bolts
# since spouts don't have parents
- for name, bolt_spec in self.bolts.iteritems():
+ for name, bolt_spec in self.bolts.items():
if name == self.TERMINAL_BOLT_NAME:
continue
@@ -144,20 +144,20 @@ class TestTopologyBuilder(TopologyBuilder):
non_terminals = set()
# 1. terminal bolts need upstream components, because we don't want isolated bolts
# 2. terminal bolts should not exist in the prev.values(), meaning that no downstream
- for parent_set in self.prev.values():
+ for parent_set in list(self.prev.values()):
non_terminals.update(parent_set)
- for bolt_name in self.prev.keys():
+ for bolt_name in list(self.prev.keys()):
if bolt_name not in non_terminals:
terminals.add(bolt_name)
# will also consider the cases with spouts without children
- for spout_name in self.spouts.keys():
+ for spout_name in list(self.spouts.keys()):
if spout_name not in non_terminals:
terminals.add(spout_name)
# add all grouping to components
- for child in self.prev.keys():
+ for child in list(self.prev.keys()):
for parent in self.prev[child]:
self._add_all_grouping(child, parent, integ_const.INTEGRATION_TEST_CONTROL_STREAM_ID)
diff --git a/integration_test/src/python/local_test_runner/test_template.py b/integration_test/src/python/local_test_runner/test_template.py
index 78fe215..bc4f86d 100644
--- a/integration_test/src/python/local_test_runner/test_template.py
+++ b/integration_test/src/python/local_test_runner/test_template.py
@@ -23,7 +23,7 @@ import json
import logging
import os
import time
-import urllib
+import urllib.request, urllib.parse, urllib.error
import signal
import subprocess
from collections import namedtuple
@@ -252,7 +252,7 @@ class TestTemplate(object):
url = 'http://localhost:%s/topologies/physicalplan?' % self.params['trackerPort']\
+ 'cluster=local&environ=default&topology=IntegrationTest_LocalReadWriteTopology'
logging.debug("Fetching physical plan from %s", url)
- response = urllib.urlopen(url)
+ response = urllib.request.urlopen(url)
physical_plan_json = json.loads(response.read())
if 'result' not in physical_plan_json:
diff --git a/integration_test/src/python/test_runner/main.py b/integration_test/src/python/test_runner/main.py
index 0d4d096..a09803a 100644
--- a/integration_test/src/python/test_runner/main.py
+++ b/integration_test/src/python/test_runner/main.py
@@ -24,7 +24,7 @@ import re
import sys
import time
import uuid
-from httplib import HTTPConnection
+from http.client import HTTPConnection
from threading import Lock, Thread
from ..common import status
@@ -82,7 +82,7 @@ class HttpBasedExpectedResultsHandler(object):
# need to convert from a list of json objects to a string of a python list,
# without the unicode using double quotes, not single quotes.
- return str(map(lambda x: str(x), result)).replace("'", '"')
+ return str([str(x) for x in result]).replace("'", '"')
except Exception as e:
raise status.TestFailure(
"Fetching expected result failed for %s topology" % self.topology_name, e)
@@ -133,8 +133,8 @@ class ExactlyOnceResultsChecker(object):
else:
failure = status.TestFailure("Actual result did not match expected result")
# lambda required below to remove the unicode 'u' from the output
- logging.info("Actual result ---------- \n" + str(map(lambda x: str(x), actual_results)))
- logging.info("Expected result ---------- \n" + str(map(lambda x: str(x), expected_results)))
+ logging.info("Actual result ---------- \n" + str([str(x) for x in actual_results]))
+ logging.info("Expected result ---------- \n" + str([str(x) for x in expected_results]))
raise failure
class AtLeastOnceResultsChecker(ExactlyOnceResultsChecker):
@@ -161,9 +161,9 @@ class AtLeastOnceResultsChecker(ExactlyOnceResultsChecker):
failure = status.TestFailure("Actual result did not match expected result")
# lambda required below to remove the unicode 'u' from the output
logging.info("Actual value frequencies ---------- \n" + ', '.join(
- map(lambda (k, v): "%s(%s)" % (str(k), v), actual_counts.iteritems())))
+ ["%s(%s)" % (str(k_v[0]), k_v[1]) for k_v in iter(actual_counts.items())]))
logging.info("Expected value frequencies ---------- \n" + ', '.join(
- map(lambda (k, v): "%s(%s)" % (str(k), v), expected_counts.iteritems())))
+ ["%s(%s)" % (str(k_v1[0]), k_v1[1]) for k_v1 in iter(expected_counts.items())]))
raise failure
def _frequency_dict(values):
@@ -301,11 +301,11 @@ def filter_test_topologies(test_topologies, test_pattern):
initial_topologies = test_topologies
if test_pattern:
pattern = re.compile(test_pattern)
- test_topologies = filter(lambda x: pattern.match(x['topologyName']), test_topologies)
+ test_topologies = [x for x in test_topologies if pattern.match(x['topologyName'])]
if len(test_topologies) == 0:
logging.error("Test filter '%s' did not match any configured test names:\n%s",
- test_pattern, '\n'.join(map(lambda x: x['topologyName'], initial_topologies)))
+ test_pattern, '\n'.join([x['topologyName'] for x in initial_topologies]))
sys.exit(1)
return test_topologies
diff --git a/integration_test/src/python/topology_test_runner/main.py b/integration_test/src/python/topology_test_runner/main.py
index 08fefea..2ba8184 100644
--- a/integration_test/src/python/topology_test_runner/main.py
+++ b/integration_test/src/python/topology_test_runner/main.py
@@ -24,7 +24,7 @@ import re
import sys
import time
import uuid
-from httplib import HTTPConnection
+from http.client import HTTPConnection
from threading import Lock, Thread
from ..common import status
@@ -198,7 +198,7 @@ class InstanceStateResultChecker(TopologyStructureResultChecker):
raise status.TestFailure("Fail to get actual results of instance states for topology %s"
% self.topology_name)
- if '_' not in expected_result[0].keys()[0]:
+ if '_' not in list(expected_result[0].keys())[0]:
actual_result = self._parse_instance_id(actual_result)
return self._compare_state(sorted(expected_result), sorted(actual_result))
@@ -210,8 +210,8 @@ class InstanceStateResultChecker(TopologyStructureResultChecker):
else:
failure = status.TestFailure("Actual result did not match expected result")
# lambda required below to remove the unicode 'u' from the output
- logging.info("Actual result ---------- \n" + str(map(lambda x: str(x), actual_results)))
- logging.info("Expected result ---------- \n" + str(map(lambda x: str(x), expected_results)))
+ logging.info("Actual result ---------- \n" + str([str(x) for x in actual_results]))
+ logging.info("Expected result ---------- \n" + str([str(x) for x in expected_results]))
raise failure
def _parse_instance_id(self, input):
@@ -345,11 +345,11 @@ def filter_test_topologies(test_topologies, test_pattern):
initial_topologies = test_topologies
if test_pattern:
pattern = re.compile(test_pattern)
- test_topologies = filter(lambda x: pattern.match(x['topologyName']), test_topologies)
+ test_topologies = [x for x in test_topologies if pattern.match(x['topologyName'])]
if len(test_topologies) == 0:
logging.error("Test filter '%s' did not match any configured test names:\n%s",
- test_pattern, '\n'.join(map(lambda x: x['topologyName'], initial_topologies)))
+ test_pattern, '\n'.join([x['topologyName'] for x in initial_topologies]))
sys.exit(1)
return test_topologies
diff --git a/scripts/shutils/save-logs.py b/scripts/shutils/save-logs.py
index f101e63..c378936 100755
--- a/scripts/shutils/save-logs.py
+++ b/scripts/shutils/save-logs.py
@@ -30,7 +30,7 @@ def tail(filename, n):
with open(filename, "rb") as f:
fm = mmap.mmap(f.fileno(), 0, mmap.MAP_SHARED, mmap.PROT_READ)
try:
- for i in xrange(size - 1, -1, -1):
+ for i in range(size - 1, -1, -1):
if fm[i] == '\n':
n -= 1
if n == -1:
@@ -40,7 +40,7 @@ def tail(filename, n):
fm.close()
def main(file, cmd):
- print "%s writing to: %s" % (cmd, file)
+ print("%s writing to: %s" % (cmd, file))
with open(file, "w") as out:
count = 0
process = subprocess.Popen(cmd,
@@ -64,16 +64,16 @@ def main(file, cmd):
errcode = process.wait()
diff = datetime.now() - start
sys.stdout.write("\r%d seconds %d log lines"%(diff.seconds, count))
- print "\n %s finished with errcode: %d" % (cmd, errcode)
+ print("\n %s finished with errcode: %d" % (cmd, errcode))
if errcode != 0:
lines = tail(file, 1000)
- print '\n'.join(lines)
+ print('\n'.join(lines))
sys.exit(errcode)
return errcode
if __name__ == "__main__":
if sys.argv < 1:
- print "Usage: %s [file info]" % sys.argv[0]
+ print("Usage: %s [file info]" % sys.argv[0])
sys.exit(1)
file = sys.argv[1]
cmd = sys.argv[2:]
diff --git a/third_party/python/cpplint/cpplint.py b/third_party/python/cpplint/cpplint.py
index 74c75bc..2e51787 100755
--- a/third_party/python/cpplint/cpplint.py
+++ b/third_party/python/cpplint/cpplint.py
@@ -585,7 +585,7 @@ _ALT_TOKEN_REPLACEMENT = {
# False positives include C-style multi-line comments and multi-line strings
# but those have always been troublesome for cpplint.
_ALT_TOKEN_REPLACEMENT_PATTERN = re.compile(
- r'[ =()](' + ('|'.join(_ALT_TOKEN_REPLACEMENT.keys())) + r')(?=[ (]|$)')
+ r'[ =()](' + ('|'.join(list(_ALT_TOKEN_REPLACEMENT.keys()))) + r')(?=[ (]|$)')
# These constants define types of headers for use with
@@ -640,19 +640,19 @@ _quiet = False
_line_length = 80
try:
- xrange(1, 0)
+ range(1, 0)
except NameError:
# -- pylint: disable=redefined-builtin
xrange = range
try:
- unicode
+ str
except NameError:
# -- pylint: disable=redefined-builtin
- basestring = unicode = str
+ str = str = str
try:
- long(2)
+ int(2)
except NameError:
# -- pylint: disable=redefined-builtin
long = int
@@ -1699,7 +1699,7 @@ def FindEndOfExpressionInLine(line, startpos, stack):
On finding an unclosed expression: (-1, None)
Otherwise: (-1, new stack at end of this line)
"""
- for i in xrange(startpos, len(line)):
+ for i in range(startpos, len(line)):
char = line[i]
if char in '([{':
# Found start of parenthesized expression, push to expression stack
@@ -2066,7 +2066,7 @@ def CheckForHeaderGuard(filename, clean_lines, error):
# contain any "//" comments at all, it could be that the compiler
# only wants "/**/" comments, look for those instead.
no_single_line_comments = True
- for i in xrange(1, len(raw_lines) - 1):
+ for i in range(1, len(raw_lines) - 1):
line = raw_lines[i]
if Match(r'^(?:(?:\'(?:\.|[^\'])*\')|(?:"(?:\.|[^"])*")|[^\'"])*//', line):
no_single_line_comments = False
@@ -2131,7 +2131,7 @@ def CheckForBadCharacters(filename, lines, error):
error: The function to call with any errors found.
"""
for linenum, line in enumerate(lines):
- if unicode_escape_decode('\ufffd') in line:
+ if unicode_escape_decode('\\ufffd') in line:
error(filename, linenum, 'readability/utf8', 5,
'Line contains invalid UTF-8 (or Unicode replacement character).')
if '\0' in line:
@@ -2409,7 +2409,7 @@ class _ClassInfo(_BlockInfo):
# If there is a DISALLOW macro, it should appear near the end of
# the class.
seen_last_thing_in_class = False
- for i in xrange(linenum - 1, self.starting_linenum, -1):
+ for i in range(linenum - 1, self.starting_linenum, -1):
match = Search(
r'\b(DISALLOW_COPY_AND_ASSIGN|DISALLOW_IMPLICIT_CONSTRUCTORS)\(' +
self.name + r'\)',
@@ -3681,7 +3681,7 @@ def _IsType(clean_lines, nesting_state, expr):
continue
# Look for typename in the specified range
- for i in xrange(first_line, last_line + 1, 1):
+ for i in range(first_line, last_line + 1, 1):
if Search(typename_pattern, clean_lines.elided[i]):
return True
block_index -= 1
@@ -3745,7 +3745,7 @@ def CheckBracesSpacing(filename, clean_lines, linenum, nesting_state, error):
trailing_text = ''
if endpos > -1:
trailing_text = endline[endpos:]
- for offset in xrange(endlinenum + 1,
+ for offset in range(endlinenum + 1,
min(endlinenum + 3, clean_lines.NumLines() - 1)):
trailing_text += clean_lines.elided[offset]
# We also suppress warnings for `uint64_t{expression}` etc., as the style
@@ -4298,7 +4298,7 @@ def CheckCheck(filename, clean_lines, linenum, error):
expression = lines[linenum][start_pos + 1:end_pos - 1]
else:
expression = lines[linenum][start_pos + 1:]
- for i in xrange(linenum + 1, end_line):
+ for i in range(linenum + 1, end_line):
expression += lines[i]
expression += last_line[0:end_pos - 1]
@@ -4426,7 +4426,7 @@ def GetLineWidth(line):
The width of the line in column positions, accounting for Unicode
combining characters and wide characters.
"""
- if isinstance(line, unicode):
+ if isinstance(line, str):
width = 0
for uc in unicodedata.normalize('NFC', line):
if unicodedata.east_asian_width(uc) in ('W', 'F'):
@@ -5096,7 +5096,7 @@ def IsDerivedFunction(clean_lines, linenum):
virt-specifier.
"""
# Scan back a few lines for start of current function
- for i in xrange(linenum, max(-1, linenum - 10), -1):
+ for i in range(linenum, max(-1, linenum - 10), -1):
match = Match(r'^([^()]*\w+)\(', clean_lines.elided[i])
if match:
# Look for "override" after the matching closing parenthesis
@@ -5117,7 +5117,7 @@ def IsOutOfLineMethodDefinition(clean_lines, linenum):
True if current line contains an out-of-line method definition.
"""
# Scan back a few lines for start of current function
- for i in xrange(linenum, max(-1, linenum - 10), -1):
+ for i in range(linenum, max(-1, linenum - 10), -1):
if Match(r'^([^()]*\w+)\(', clean_lines.elided[i]):
return Match(r'^[^()]*\w+::\w+\(', clean_lines.elided[i]) is not None
return False
@@ -5133,7 +5133,7 @@ def IsInitializerList(clean_lines, linenum):
True if current line appears to be inside constructor initializer
list, False otherwise.
"""
- for i in xrange(linenum, 1, -1):
+ for i in range(linenum, 1, -1):
line = clean_lines.elided[i]
if i == linenum:
remove_function_body = Match(r'^(.*)\{\s*$', line)
@@ -5234,7 +5234,7 @@ def CheckForNonConstReference(filename, clean_lines, linenum,
# Found the matching < on an earlier line, collect all
# pieces up to current line.
line = ''
- for i in xrange(startline, linenum + 1):
+ for i in range(startline, linenum + 1):
line += clean_lines.elided[i].strip()
# Check for non-const references in function parameters. A single '&' may
@@ -5258,7 +5258,7 @@ def CheckForNonConstReference(filename, clean_lines, linenum,
# appear inside the second set of parentheses on the current line as
# opposed to the first set.
if linenum > 0:
- for i in xrange(linenum - 1, max(0, linenum - 10), -1):
+ for i in range(linenum - 1, max(0, linenum - 10), -1):
previous_line = clean_lines.elided[i]
if not Search(r'[),]\s*$', previous_line):
break
@@ -5289,7 +5289,7 @@ def CheckForNonConstReference(filename, clean_lines, linenum,
# Don't see a whitelisted function on this line. Actually we
# didn't see any function name on this line, so this is likely a
# multi-line parameter list. Try a bit harder to catch this case.
- for i in xrange(2):
+ for i in range(2):
if (linenum > i and
Search(whitelisted_functions, clean_lines.elided[linenum - i - 1])):
return
@@ -5452,7 +5452,7 @@ def CheckCStyleCast(filename, clean_lines, linenum, cast_type, pattern, error):
# Try expanding current context to see if we one level of
# parentheses inside a macro.
if linenum > 0:
- for i in xrange(linenum - 1, max(0, linenum - 5), -1):
+ for i in range(linenum - 1, max(0, linenum - 5), -1):
context = clean_lines.elided[i] + context
if Match(r'.*\b[_A-Z][_A-Z0-9]*\s*\((?:\([^()]*\)|[^()])*$', context):
return False
@@ -5804,7 +5804,7 @@ def CheckRedundantVirtual(filename, clean_lines, linenum, error):
end_col = -1
end_line = -1
start_col = len(virtual.group(2))
- for start_line in xrange(linenum, min(linenum + 3, clean_lines.NumLines())):
+ for start_line in range(linenum, min(linenum + 3, clean_lines.NumLines())):
line = clean_lines.elided[start_line][start_col:]
parameter_list = Match(r'^([^(]*)\(', line)
if parameter_list:
@@ -5819,7 +5819,7 @@ def CheckRedundantVirtual(filename, clean_lines, linenum, error):
# Look for "override" or "final" after the parameter list
# (possibly on the next few lines).
- for i in xrange(end_line, min(end_line + 3, clean_lines.NumLines())):
+ for i in range(end_line, min(end_line + 3, clean_lines.NumLines())):
line = clean_lines.elided[i][end_col:]
match = Search(r'\b(override|final)\b', line)
if match: