You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@allura.apache.org by tv...@apache.org on 2013/02/05 21:23:34 UTC
[35/42] git commit: [#4691] Cleaned up ModelCache debugging counters
[#4691] Cleaned up ModelCache debugging counters
Signed-off-by: Cory Johns <jo...@geek.net>
Project: http://git-wip-us.apache.org/repos/asf/incubator-allura/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-allura/commit/c340d113
Tree: http://git-wip-us.apache.org/repos/asf/incubator-allura/tree/c340d113
Diff: http://git-wip-us.apache.org/repos/asf/incubator-allura/diff/c340d113
Branch: refs/heads/master
Commit: c340d1134b9b1a036aed53916c2a0286329c0edd
Parents: 7fd94b9
Author: Cory Johns <jo...@geek.net>
Authored: Tue Jan 29 22:16:39 2013 +0000
Committer: Tim Van Steenburgh <tv...@gmail.com>
Committed: Tue Feb 5 20:22:52 2013 +0000
----------------------------------------------------------------------
Allura/allura/model/repo.py | 19 ---------------
Allura/allura/model/repo_refresh.py | 2 -
scripts/refresh-last-commits.py | 36 ++++-------------------------
3 files changed, 5 insertions(+), 52 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/c340d113/Allura/allura/model/repo.py
----------------------------------------------------------------------
diff --git a/Allura/allura/model/repo.py b/Allura/allura/model/repo.py
index b90b0c7..0f4352c 100644
--- a/Allura/allura/model/repo.py
+++ b/Allura/allura/model/repo.py
@@ -819,21 +819,6 @@ class ModelCache(object):
self._synthetic_ids = defaultdict(set)
self._synthetic_id_queries = defaultdict(set)
- # temporary, for performance testing
- self._query_hits = defaultdict(int)
- self._instance_hits = defaultdict(int)
- self._accesses = defaultdict(int)
- self._get_calls = 0
- self._get_walks = 0
- self._get_walks_max = 0
- self._get_hits = 0
- self._get_misses = 0
- self._build_calls = 0
- self._build_walks = 0
- self._build_walks_max = 0
- self._skipped_trees = 0
- self._processed_trees = 0
-
def _normalize_query(self, query):
_query = query
if not isinstance(_query, tuple):
@@ -849,23 +834,19 @@ class ModelCache(object):
raise AttributeError('%s has neither "query" nor "m" attribute' % cls)
def get(self, cls, query):
- self._accesses[cls] += 1
_query = self._normalize_query(query)
self._touch(cls, _query)
if _query not in self._query_cache[cls]:
val = self._model_query(cls).get(**query)
self.set(cls, _query, val)
return val
- self._query_hits[cls] += 1
_id = self._query_cache[cls][_query]
if _id is None:
- self._instance_hits[cls] += 1
return None
if _id not in self._instance_cache[cls]:
val = self._model_query(cls).get(**query)
self.set(cls, _query, val)
return val
- self._instance_hits[cls] += 1
return self._instance_cache[cls][_id]
def set(self, cls, query, val):
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/c340d113/Allura/allura/model/repo_refresh.py
----------------------------------------------------------------------
diff --git a/Allura/allura/model/repo_refresh.py b/Allura/allura/model/repo_refresh.py
index 4bbd6e1..bdd5a1f 100644
--- a/Allura/allura/model/repo_refresh.py
+++ b/Allura/allura/model/repo_refresh.py
@@ -499,9 +499,7 @@ def compute_lcds(commit, cache):
_compute_lcds(tree, cache)
def _compute_lcds(tree, cache):
- cache._processed_trees += 1
if tree.path().strip('/') not in tree.commit.changed_paths:
- cache._skipped_trees += 1
return
lcd = LastCommit.get(tree, create=True) # auto-vivify LCD
for x in tree.tree_ids:
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/c340d113/scripts/refresh-last-commits.py
----------------------------------------------------------------------
diff --git a/scripts/refresh-last-commits.py b/scripts/refresh-last-commits.py
index b17a670..5b6e78a 100644
--- a/scripts/refresh-last-commits.py
+++ b/scripts/refresh-last-commits.py
@@ -93,11 +93,7 @@ def refresh_repo_lcds(commit_ids, options):
with time(timings):
M.repo_refresh.compute_diffs(c.app.repo._id, tree_cache, commit)
if len(timings) % 1000 == 0:
- mt = max(timings)
- tt = sum(timings)
- at = tt / len(timings)
- print ' Processed %d commits (max: %f, avg: %f, tot: %f, cl: %d)' % (
- len(timings), mt, at, tt, len(tree_cache))
+ _print_stats(timings, 1000)
lcd_cache = M.repo.ModelCache(
max_instances={M.repo.LastCommit: 4000},
max_queries={M.repo.LastCommit: 10000},
@@ -116,39 +112,17 @@ def refresh_repo_lcds(commit_ids, options):
# so that every commit sees the same copy
lcd_cache.expire_new_instances(M.repo.LastCommit)
if len(timings) % debug_step == 0:
- _print_stats(lcd_cache, timings, debug_step, commit)
- lcd_cache._get_walks_max = 0
- lcd_cache._build_walks_max = 0
+ _print_stats(timings, debug_step)
ThreadLocalORMSession.flush_all()
-def _print_stats(lcd_cache, timings, debug_step, commit):
+def _print_stats(timings, debug_step):
mt = max(timings)
tt = sum(timings)
at = tt / len(timings)
mat = sum(timings[-debug_step:]) / debug_step
- laccs = lcd_cache._accesses[M.repo.LastCommit]
- lqhits = lcd_cache._query_hits[M.repo.LastCommit]
- lqavg = lqhits * 100 / laccs if laccs > 0 else 0
- lihits = lcd_cache._instance_hits[M.repo.LastCommit]
- liavg = lihits * 100 / laccs if laccs > 0 else 0
- oaccs = sum([v for k,v in lcd_cache._accesses.items() if k != M.repo.LastCommit])
- oqhits = sum([v for k,v in lcd_cache._query_hits.items() if k != M.repo.LastCommit])
- oqavg = oqhits * 100 / oaccs if oaccs > 0 else 0
- oihits = sum([v for k,v in lcd_cache._instance_hits.items() if k != M.repo.LastCommit])
- oiavg = oihits * 100 / oaccs if oaccs > 0 else 0
- gper = lcd_cache._get_hits * 100 / lcd_cache._get_calls if lcd_cache._get_calls > 0 else 0
- print ' Processed %d commits (max: %f avg: %f mavg: %f tot: %f lq: %d/%d li: %d/%d qhit: %d/%d ihit: %d/%d mgw: %d gh: %d mbw: %d ts: %d ps: %d)' % (
- len(timings), mt, at, mat, tt,
- lcd_cache.num_queries(), lcd_cache.num_queries(M.repo.LastCommit),
- lcd_cache.num_instances(), lcd_cache.num_instances(M.repo.LastCommit),
- oqavg, lqavg,
- oiavg, liavg,
- lcd_cache._get_walks_max, gper,
- lcd_cache._build_walks_max,
- len(lcd_cache.get(M.repo.TreesDoc, dict(_id=commit._id)).tree_ids),
- lcd_cache._skipped_trees * 100 / lcd_cache._processed_trees)
-
+ print ' Processed %d commits (max: %f, avg: %f, mavg: %f, tot: %f)' % (
+ len(timings), mt, at, mat, tt)
@contextmanager
def time(timings):