You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@allura.apache.org by br...@apache.org on 2013/04/24 23:53:26 UTC

[1/2] git commit: [#6078] Chunk processing of commits when cleaning data during repo refresh to avoid BSON doc size limits

Updated Branches:
  refs/heads/master 51f5361f0 -> 5f980b72b


[#6078] Chunk processing of commits when cleaning data during repo refresh to avoid BSON doc size limits

Signed-off-by: Cory Johns <cj...@slashdotmedia.com>


Project: http://git-wip-us.apache.org/repos/asf/incubator-allura/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-allura/commit/115cdad1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-allura/tree/115cdad1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-allura/diff/115cdad1

Branch: refs/heads/master
Commit: 115cdad111a046102abf4d8063482c6c6bf0c615
Parents: 51f5361
Author: Cory Johns <cj...@slashdotmedia.com>
Authored: Thu Apr 11 16:18:12 2013 +0000
Committer: Dave Brondsema <db...@slashdotmedia.com>
Committed: Wed Apr 24 21:37:34 2013 +0000

----------------------------------------------------------------------
 Allura/allura/scripts/refreshrepo.py |   66 +++++++++++++++++------------
 1 files changed, 39 insertions(+), 27 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/115cdad1/Allura/allura/scripts/refreshrepo.py
----------------------------------------------------------------------
diff --git a/Allura/allura/scripts/refreshrepo.py b/Allura/allura/scripts/refreshrepo.py
index 43ff7c2..f55a054 100644
--- a/Allura/allura/scripts/refreshrepo.py
+++ b/Allura/allura/scripts/refreshrepo.py
@@ -68,42 +68,54 @@ class RefreshRepo(ScriptTask):
                     if options.clean:
                         ci_ids = list(c.app.repo.all_commit_ids())
                         log.info("Deleting mongo data for %i commits...", len(ci_ids))
-                        tree_ids = [
-                                tree_id for doc in
-                                M.repo.TreesDoc.m.find({"_id": {"$in": ci_ids}},
-                                                       {"tree_ids": 1})
-                                for tree_id in doc.get("tree_ids", [])]
-
-                        i = M.repo.CommitDoc.m.find({"_id": {"$in": ci_ids}}).count()
-                        log.info("Deleting %i CommitDoc docs...", i)
-                        M.repo.CommitDoc.m.remove({"_id": {"$in": ci_ids}})
+                        # like the tree_ids themselves below, we need to process these in
+                        # chunks to avoid hitting the BSON max size limit
+                        tree_ids = []
+                        for ci_ids_chunk in chunked_list(ci_ids, 3000):
+                            tree_ids.extend([
+                                    tree_id for doc in
+                                    M.repo.TreesDoc.m.find({"_id": {"$in": ci_ids_chunk}},
+                                                           {"tree_ids": 1})
+                                    for tree_id in doc.get("tree_ids", [])])
+
+                            i = M.repo.CommitDoc.m.find({"_id": {"$in": ci_ids_chunk}}).count()
+                            if i:
+                                log.info("Deleting %i CommitDoc docs...", i)
+                                M.repo.CommitDoc.m.remove({"_id": {"$in": ci_ids_chunk}})
 
                         # delete these in chunks, otherwise the query doc can
                         # exceed the max BSON size limit (16MB at the moment)
                         for tree_ids_chunk in chunked_list(tree_ids, 300000):
                             i = M.repo.TreeDoc.m.find({"_id": {"$in": tree_ids_chunk}}).count()
-                            log.info("Deleting %i TreeDoc docs...", i)
-                            M.repo.TreeDoc.m.remove({"_id": {"$in": tree_ids_chunk}})
+                            if i:
+                                log.info("Deleting %i TreeDoc docs...", i)
+                                M.repo.TreeDoc.m.remove({"_id": {"$in": tree_ids_chunk}})
                         del tree_ids
 
                         # delete these after TreeDoc and LastCommitDoc so that if
                         # we crash, we don't lose the ability to delete those
-                        i = M.repo.TreesDoc.m.find({"_id": {"$in": ci_ids}}).count()
-                        log.info("Deleting %i TreesDoc docs...", i)
-                        M.repo.TreesDoc.m.remove({"_id": {"$in": ci_ids}})
-
-                        # delete LastCommitDocs
-                        i = M.repo.LastCommitDoc.m.find(dict(commit_ids={'$in': ci_ids})).count()
-                        log.info("Deleting %i remaining LastCommitDoc docs, by repo id...", i)
-                        M.repo.LastCommitDoc.m.remove(dict(commit_ids={'$in': ci_ids}))
-
-                        i = M.repo.DiffInfoDoc.m.find({"_id": {"$in": ci_ids}}).count()
-                        log.info("Deleting %i DiffInfoDoc docs...", i)
-                        M.repo.DiffInfoDoc.m.remove({"_id": {"$in": ci_ids}})
-
-                        i = M.repo.CommitRunDoc.m.find({"commit_ids": {"$in": ci_ids}}).count()
-                        log.info("Deleting %i CommitRunDoc docs...", i)
-                        M.repo.CommitRunDoc.m.remove({"commit_ids": {"$in": ci_ids}})
+                        for ci_ids_chunk in chunked_list(ci_ids, 3000):
+                            # delete TreesDocs
+                            i = M.repo.TreesDoc.m.find({"_id": {"$in": ci_ids_chunk}}).count()
+                            if i:
+                                log.info("Deleting %i TreesDoc docs...", i)
+                                M.repo.TreesDoc.m.remove({"_id": {"$in": ci_ids_chunk}})
+
+                            # delete LastCommitDocs
+                            i = M.repo.LastCommitDoc.m.find(dict(commit_ids={'$in': ci_ids_chunk})).count()
+                            if i:
+                                log.info("Deleting %i remaining LastCommitDoc docs, by repo id...", i)
+                                M.repo.LastCommitDoc.m.remove(dict(commit_ids={'$in': ci_ids_chunk}))
+
+                            i = M.repo.DiffInfoDoc.m.find({"_id": {"$in": ci_ids_chunk}}).count()
+                            if i:
+                                log.info("Deleting %i DiffInfoDoc docs...", i)
+                                M.repo.DiffInfoDoc.m.remove({"_id": {"$in": ci_ids_chunk}})
+
+                            i = M.repo.CommitRunDoc.m.find({"commit_ids": {"$in": ci_ids_chunk}}).count()
+                            if i:
+                                log.info("Deleting %i CommitRunDoc docs...", i)
+                                M.repo.CommitRunDoc.m.remove({"commit_ids": {"$in": ci_ids_chunk}})
                         del ci_ids
 
                     try:


[2/2] git commit: [#6078] fix gitpython URL with our patches

Posted by br...@apache.org.
[#6078] fix gitpython URL with our patches


Project: http://git-wip-us.apache.org/repos/asf/incubator-allura/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-allura/commit/5f980b72
Tree: http://git-wip-us.apache.org/repos/asf/incubator-allura/tree/5f980b72
Diff: http://git-wip-us.apache.org/repos/asf/incubator-allura/diff/5f980b72

Branch: refs/heads/master
Commit: 5f980b72ba2acc182999a5bb3b43aacc074660f3
Parents: 115cdad
Author: Dave Brondsema <db...@slashdotmedia.com>
Authored: Mon Apr 15 20:41:58 2013 +0000
Committer: Dave Brondsema <db...@slashdotmedia.com>
Committed: Wed Apr 24 21:39:40 2013 +0000

----------------------------------------------------------------------
 requirements-sf.txt |    2 +-
 1 files changed, 1 insertions(+), 1 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/5f980b72/requirements-sf.txt
----------------------------------------------------------------------
diff --git a/requirements-sf.txt b/requirements-sf.txt
index ed3e552..0923185 100644
--- a/requirements-sf.txt
+++ b/requirements-sf.txt
@@ -17,7 +17,7 @@ wsgipreload==1.2
 pyzmq==2.1.7
 html2text==3.200.3dev-20121112
 
-# use version built from https://github.com/johnsca/GitPython/commits/0.3.2-RC1-20130424
+# use version built from https://github.com/johnsca/GitPython/commits/tv/6000
 # for unmerged fixes for [#5411], [#6000], and [#6078]
 GitPython==0.3.2.RC1-20130424