You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tvm.apache.org by GitBox <gi...@apache.org> on 2022/09/15 22:13:27 UTC
[GitHub] [tvm] areusch commented on a diff in pull request #12522: [AutoTVM] [TVMC] Visualize tuning progress
areusch commented on code in PR #12522:
URL: https://github.com/apache/tvm/pull/12522#discussion_r972455828
##########
python/tvm/auto_scheduler/task_scheduler.py:
##########
@@ -659,3 +660,87 @@ def post_tune(self, task_scheduler, task_id):
)
)
filep.flush()
+
+
+class VisualizeProgress(TaskSchedulerCallback):
+ """Callback that generates a plot to visualize the tuning progress.
+
+ Parameters
+ ----------
+ title: str
+ Specify the title of the matplotlib figure.
+ si_prefix: str
+ SI prefix for flops
+ keep_open: bool
+ Wait until the matplotlib window was closed by the user.
+ live: bool
+ If false, the graph is only written to the file specified in out_path.
+ out_path: str
+ Path where the graph image should be written (if defined).
+ """
+
+ def __init__(
+ self,
+ title="AutoScheduler Progress",
+ si_prefix="G",
+ keep_open=False,
+ live=True,
+ out_path=None,
+ ):
+ self.best_flops = {}
+ self.all_cts = {}
+ self.si_prefix = si_prefix
+ self.keep_open = keep_open
+ self.live = live
+ self.out_path = out_path
+ self.init_plot(title)
+
+ def __del__(self):
+ import matplotlib.pyplot as plt
+
+ if self.out_path:
+ print(f"Writing plot to file {self.out_path}...")
+ plt.savefig(self.out_path)
+ if self.live and self.keep_open:
+ print("Close matplotlib window to continue...")
+ plt.show()
+
+ def pre_tune(self, task_scheduler, task_id):
+ for i, task in enumerate(task_scheduler.tasks):
+ cts = task_scheduler.task_cts[i]
+ if cts == 0:
+ self.all_cts[i] = [0]
+ self.best_flops[i] = [0]
+ self.update_plot()
+
+ def post_tune(self, task_scheduler, task_id):
+ for i, task in enumerate(task_scheduler.tasks):
+ if task_scheduler.best_costs[i] < 1e9:
+ flops = task_scheduler.tasks[i].compute_dag.flop_ct / task_scheduler.best_costs[i]
+ flops = format_si_prefix(flops, self.si_prefix)
+ cts = task_scheduler.task_cts[i]
+ if cts not in self.all_cts[i]:
+ self.all_cts[i].append(cts)
+ self.best_flops[i].append(flops)
+ else:
+ flops = None
+ self.update_plot()
+
+ def init_plot(self, title):
+ import matplotlib.pyplot as plt
+
+ plt.figure(title)
+
+ def update_plot(self):
+ import matplotlib.pyplot as plt
+
+ plt.clf()
+ for i in range(len(self.all_cts)):
+ if i in self.all_cts and i in self.best_flops:
+ color = plt.cm.tab10(i)
+ plt.plot(self.all_cts[i], self.best_flops[i], color=color, label=f"Task {i}")
+ plt.legend(loc="upper left")
+ plt.xlabel("Iterations")
+ plt.ylabel(f"{self.si_prefix}FLOPS")
+ if self.live:
+ plt.pause(0.05)
Review Comment:
can this be 0?
##########
python/tvm/auto_scheduler/task_scheduler.py:
##########
@@ -659,3 +660,87 @@ def post_tune(self, task_scheduler, task_id):
)
)
filep.flush()
+
+
+class VisualizeProgress(TaskSchedulerCallback):
+ """Callback that generates a plot to visualize the tuning progress.
+
+ Parameters
+ ----------
+ title: str
+ Specify the title of the matplotlib figure.
+ si_prefix: str
+ SI prefix for flops
+ keep_open: bool
+ Wait until the matplotlib window was closed by the user.
+ live: bool
+ If false, the graph is only written to the file specified in out_path.
+ out_path: str
+ Path where the graph image should be written (if defined).
+ """
+
+ def __init__(
+ self,
+ title="AutoScheduler Progress",
+ si_prefix="G",
+ keep_open=False,
+ live=True,
+ out_path=None,
+ ):
+ self.best_flops = {}
+ self.all_cts = {}
+ self.si_prefix = si_prefix
+ self.keep_open = keep_open
+ self.live = live
+ self.out_path = out_path
+ self.init_plot(title)
+
+ def __del__(self):
Review Comment:
we shouldn't do all this stuff in `__del__`. Tracebacks aren't reported and it can be unpredictable when this is invoked. Can you change this to use `__enter__` and `__exit__` so it can be used with `with` statement, and add an explicit part in the `tvmc` tuning infra where we expect this file written?
##########
python/tvm/auto_scheduler/task_scheduler.py:
##########
@@ -659,3 +660,87 @@ def post_tune(self, task_scheduler, task_id):
)
)
filep.flush()
+
+
+class VisualizeProgress(TaskSchedulerCallback):
+ """Callback that generates a plot to visualize the tuning progress.
+
+ Parameters
+ ----------
+ title: str
+ Specify the title of the matplotlib figure.
+ si_prefix: str
+ SI prefix for flops
+ keep_open: bool
+ Wait until the matplotlib window was closed by the user.
+ live: bool
+ If false, the graph is only written to the file specified in out_path.
+ out_path: str
+ Path where the graph image should be written (if defined).
+ """
+
+ def __init__(
+ self,
+ title="AutoScheduler Progress",
+ si_prefix="G",
+ keep_open=False,
+ live=True,
+ out_path=None,
+ ):
+ self.best_flops = {}
+ self.all_cts = {}
+ self.si_prefix = si_prefix
+ self.keep_open = keep_open
+ self.live = live
+ self.out_path = out_path
+ self.init_plot(title)
+
+ def __del__(self):
+ import matplotlib.pyplot as plt
+
+ if self.out_path:
+ print(f"Writing plot to file {self.out_path}...")
+ plt.savefig(self.out_path)
+ if self.live and self.keep_open:
+ print("Close matplotlib window to continue...")
+ plt.show()
+
+ def pre_tune(self, task_scheduler, task_id):
+ for i, task in enumerate(task_scheduler.tasks):
+ cts = task_scheduler.task_cts[i]
+ if cts == 0:
+ self.all_cts[i] = [0]
+ self.best_flops[i] = [0]
+ self.update_plot()
+
+ def post_tune(self, task_scheduler, task_id):
+ for i, task in enumerate(task_scheduler.tasks):
+ if task_scheduler.best_costs[i] < 1e9:
+ flops = task_scheduler.tasks[i].compute_dag.flop_ct / task_scheduler.best_costs[i]
+ flops = format_si_prefix(flops, self.si_prefix)
+ cts = task_scheduler.task_cts[i]
+ if cts not in self.all_cts[i]:
+ self.all_cts[i].append(cts)
+ self.best_flops[i].append(flops)
+ else:
+ flops = None
+ self.update_plot()
Review Comment:
should we always update or only every N?
##########
python/tvm/auto_scheduler/task_scheduler.py:
##########
@@ -659,3 +660,87 @@ def post_tune(self, task_scheduler, task_id):
)
)
filep.flush()
+
+
+class VisualizeProgress(TaskSchedulerCallback):
+ """Callback that generates a plot to visualize the tuning progress.
+
+ Parameters
+ ----------
+ title: str
+ Specify the title of the matplotlib figure.
+ si_prefix: str
+ SI prefix for flops
+ keep_open: bool
+ Wait until the matplotlib window was closed by the user.
+ live: bool
+ If false, the graph is only written to the file specified in out_path.
+ out_path: str
+ Path where the graph image should be written (if defined).
+ """
+
+ def __init__(
+ self,
+ title="AutoScheduler Progress",
+ si_prefix="G",
+ keep_open=False,
+ live=True,
+ out_path=None,
+ ):
+ self.best_flops = {}
+ self.all_cts = {}
+ self.si_prefix = si_prefix
+ self.keep_open = keep_open
+ self.live = live
+ self.out_path = out_path
+ self.init_plot(title)
+
+ def __del__(self):
+ import matplotlib.pyplot as plt
+
+ if self.out_path:
+ print(f"Writing plot to file {self.out_path}...")
+ plt.savefig(self.out_path)
+ if self.live and self.keep_open:
+ print("Close matplotlib window to continue...")
+ plt.show()
+
+ def pre_tune(self, task_scheduler, task_id):
+ for i, task in enumerate(task_scheduler.tasks):
+ cts = task_scheduler.task_cts[i]
+ if cts == 0:
+ self.all_cts[i] = [0]
+ self.best_flops[i] = [0]
+ self.update_plot()
+
+ def post_tune(self, task_scheduler, task_id):
+ for i, task in enumerate(task_scheduler.tasks):
+ if task_scheduler.best_costs[i] < 1e9:
+ flops = task_scheduler.tasks[i].compute_dag.flop_ct / task_scheduler.best_costs[i]
+ flops = format_si_prefix(flops, self.si_prefix)
+ cts = task_scheduler.task_cts[i]
+ if cts not in self.all_cts[i]:
+ self.all_cts[i].append(cts)
+ self.best_flops[i].append(flops)
+ else:
+ flops = None
+ self.update_plot()
+
+ def init_plot(self, title):
+ import matplotlib.pyplot as plt
Review Comment:
can this be moved to the top of the file?
##########
python/tvm/driver/tvmc/autotuner.py:
##########
@@ -139,6 +138,11 @@ def add_tune_parser(subparsers, _, json_params):
help="enable tuning the graph through the AutoScheduler tuner",
action="store_true",
)
+ parser.add_argument(
+ "--visualize",
Review Comment:
this one reads like a boolean option (e.g. action="store_true"). could you better document the behavior?
##########
python/tvm/autotvm/tuner/callback.py:
##########
@@ -178,3 +178,82 @@ def _callback(tuner, inputs, results):
sys.stdout.flush()
return _callback
+
+
+def visualize_progress(
+ idx, title="AutoTVM Progress", si_prefix="G", keep_open=False, live=True, out_path=None
+):
+ """Display tuning progress in graph
+
+ Parameters
+ ----------
+ idx: int
+ Index of the current task.
+ title: str
+ Specify the title of the matplotlib figure.
+ si_prefix: str
+ SI prefix for flops
+ keep_open: bool
+ Wait until the matplotlib window was closed by the user.
+ live: bool
+ If false, the graph is only written to the file specified in out_path.
+ out_path: str
+ Path where the graph image should be written (if defined).
+ """
+ import matplotlib.pyplot as plt
+
+ class _Context(object):
+ """Context to store local variables"""
+
+ def __init__(self):
+ self.keep_open = keep_open
+ self.live = live
+ self.out_path = out_path
+ self.best_flops = [0]
+ self.all_flops = []
+ if idx > 0:
+ plt.figure(title)
+ else:
+ plt.figure(title).clear()
+ self.color = plt.cm.tab10(idx)
+ (self.p,) = plt.plot([0], [0], color=self.color, label=f"Task {idx}")
+ plt.xlabel("Iterations")
+ plt.ylabel(f"{si_prefix}FLOPS")
+ plt.legend(loc="upper left")
+ if self.live:
+ plt.pause(0.05)
+
+ def __del__(self):
Review Comment:
same request here
##########
python/tvm/autotvm/tuner/callback.py:
##########
@@ -178,3 +178,82 @@ def _callback(tuner, inputs, results):
sys.stdout.flush()
return _callback
+
+
+def visualize_progress(
+ idx, title="AutoTVM Progress", si_prefix="G", keep_open=False, live=True, out_path=None
+):
+ """Display tuning progress in graph
+
+ Parameters
+ ----------
+ idx: int
+ Index of the current task.
+ title: str
+ Specify the title of the matplotlib figure.
+ si_prefix: str
+ SI prefix for flops
+ keep_open: bool
+ Wait until the matplotlib window was closed by the user.
+ live: bool
+ If false, the graph is only written to the file specified in out_path.
+ out_path: str
+ Path where the graph image should be written (if defined).
+ """
+ import matplotlib.pyplot as plt
+
+ class _Context(object):
Review Comment:
possible to reuse any of the MS implementation?
##########
python/tvm/autotvm/tuner/callback.py:
##########
@@ -178,3 +178,82 @@ def _callback(tuner, inputs, results):
sys.stdout.flush()
return _callback
+
+
+def visualize_progress(
+ idx, title="AutoTVM Progress", si_prefix="G", keep_open=False, live=True, out_path=None
+):
+ """Display tuning progress in graph
+
+ Parameters
+ ----------
+ idx: int
+ Index of the current task.
+ title: str
+ Specify the title of the matplotlib figure.
+ si_prefix: str
+ SI prefix for flops
+ keep_open: bool
+ Wait until the matplotlib window was closed by the user.
+ live: bool
+ If false, the graph is only written to the file specified in out_path.
+ out_path: str
+ Path where the graph image should be written (if defined).
+ """
+ import matplotlib.pyplot as plt
+
+ class _Context(object):
Review Comment:
i'm not sure you need a class here to store the vars. they should be captured into the closure below if you reference them from there
##########
python/tvm/driver/tvmc/autotuner.py:
##########
@@ -638,6 +685,9 @@ def tune_tasks(
trials: int,
early_stopping: Optional[int] = None,
tuning_records: Optional[str] = None,
+ si_prefix: str = "G",
+ visualize_mode: str = "none",
Review Comment:
here as well
##########
python/tvm/driver/tvmc/autotuner.py:
##########
@@ -596,6 +636,8 @@ def schedule_tasks(
tuning_options: auto_scheduler.TuningOptions,
prior_records: Optional[str] = None,
log_estimated_latency: bool = False,
+ visualize_mode: str = "none",
Review Comment:
can you update this type to reflect the Enum class?
##########
python/tvm/driver/tvmc/autotuner.py:
##########
@@ -148,20 +152,17 @@ def add_tune_parser(subparsers, _, json_params):
auto_scheduler_group.add_argument(
"--cache-line-bytes",
type=int,
- help="the size of cache line in bytes. "
- "If not specified, it will be autoset for the current machine.",
+ help="the size of cache line in bytes. " "If not specified, it will be autoset for the current machine.",
Review Comment:
can you join the strings here and below?
##########
python/tvm/driver/tvmc/autotuner.py:
##########
@@ -228,6 +226,30 @@ def add_tune_parser(subparsers, _, json_params):
parser.set_defaults(**one_entry)
+def parse_visualize_arg(value):
+ live = False
+ file = None
+ if not value:
+ return "none", None
+ splitted = value.split(",")
+ assert len(splitted) <= 2, "The --visualize argument does not accept more than two arguments"
+ for item in splitted:
+ assert len(item) > 0, "The arguments of --visualize can not be an empty string"
+ if item == "live":
+ live = True
+ else:
+ assert file is None, "Only a single path can be passed to the --visualize argument"
+ file = pathlib.Path(item)
+ mode = "none"
Review Comment:
can you create an Enum class to document mode?
##########
python/tvm/driver/tvmc/autotuner.py:
##########
@@ -691,12 +747,25 @@ def tune_tasks(
tuner_obj.load_history(autotvm.record.load_from_file(tuning_records))
logging.info("loaded history in %.2f sec(s)", time.time() - start_time)
+ callbacks = [
+ autotvm.callback.progress_bar(trials, prefix=prefix, si_prefix=si_prefix),
+ autotvm.callback.log_to_file(log_file),
+ ]
+
+ if visualize_mode != "none":
+ assert visualize_mode in ["both", "live", "none"]
Review Comment:
update this to `assert visualize_mode in VisualizeModes`
##########
tests/python/driver/tvmc/test_autotuner.py:
##########
@@ -182,3 +182,18 @@ def test_tune_rpc_tracker_parsing(mock_load_model, mock_tune_model, mock_auto_sc
assert "10.0.0.1" == kwargs["hostname"]
assert "port" in kwargs
assert 9999 == kwargs["port"]
+
Review Comment:
possible to add a test using e.g. pyplot svg backend and a golden svg file?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@tvm.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org