svn commit: r1481458 - in /subversion/branches/fsfs-format7/subversion/libsvn_fs_fs: changes.c pack.c

[st...@apache.org]

Author: stefan2
Date: Sat May 11 23:44:08 2013
New Revision: 1481458

URL: http://svn.apache.org/r1481458
Log:
On the fsfs-format7 branch: Tune the changed path list packing code
to try harder to put as many lists into a single container as possible.

Due to the awesome compression ratios of 1:100 for certain patterns,
we significantly over-estimated the on-disk size of the container.
Now, we will serialize the container when we might have hit the block
size limit and see how much space will *actually* be left.

* subversion/libsvn_fs_fs/changes.c
  (svn_fs_fs__changes_estimate_size): 

* subversion/libsvn_fs_fs/caching.c
  (dump_cache_statistics): give a tighter estimate (still grossly over-
                           estimating in most cases)

Modified:
    subversion/branches/fsfs-format7/subversion/libsvn_fs_fs/changes.c
    subversion/branches/fsfs-format7/subversion/libsvn_fs_fs/pack.c

Modified: subversion/branches/fsfs-format7/subversion/libsvn_fs_fs/changes.c
URL: http://svn.apache.org/viewvc/subversion/branches/fsfs-format7/subversion/libsvn_fs_fs/changes.c?rev=1481458&r1=1481457&r2=1481458&view=diff
==============================================================================
--- subversion/branches/fsfs-format7/subversion/libsvn_fs_fs/changes.c (original)
+++ subversion/branches/fsfs-format7/subversion/libsvn_fs_fs/changes.c Sat May 11 23:44:08 2013
@@ -219,10 +219,10 @@ svn_fs_fs__changes_estimate_size(const s
     return 0;
 
   /* string table code makes its own prediction,
-   * changes should be < 20 bytes each,
+   * changes should be < 10 bytes each,
    * some static overhead should be assumed */
   return svn_fs_fs__string_table_builder_estimate_size(changes->builder)
-       + changes->changes->nelts * 20
+       + changes->changes->nelts * 10
        + 100;
 }
 

Modified: subversion/branches/fsfs-format7/subversion/libsvn_fs_fs/pack.c
URL: http://svn.apache.org/viewvc/subversion/branches/fsfs-format7/subversion/libsvn_fs_fs/pack.c?rev=1481458&r1=1481457&r2=1481458&view=diff
==============================================================================
--- subversion/branches/fsfs-format7/subversion/libsvn_fs_fs/pack.c (original)
+++ subversion/branches/fsfs-format7/subversion/libsvn_fs_fs/pack.c Sat May 11 23:44:08 2013
@@ -1268,6 +1268,8 @@ write_changes_containers(pack_context_t 
   int i;
 
   apr_ssize_t block_left = get_block_left(context);
+  apr_ssize_t estimated_addition = 0;
+
   svn_fs_fs__changes_t *container
     = svn_fs_fs__changes_create(1000, container_pool);
   apr_array_header_t *sub_items
@@ -1285,11 +1287,30 @@ write_changes_containers(pack_context_t 
       svn_fs_fs__p2l_entry_t *entry
         = APR_ARRAY_IDX(entries, i, svn_fs_fs__p2l_entry_t *);
 
-      if (block_left < entry->size)
-        block_left = get_block_left(context)
-                   - svn_fs_fs__changes_estimate_size(container);
+      /* zip compression alone will significantly reduce the size of large
+       * change lists. So, we will probably need even less than this estimate.
+       */
+      apr_ssize_t estimated_size = (entry->size / 5) + 250;
+
+      /* If necessary and enough data has been added to the container since
+       * the last test, try harder by actually serializing the container and
+       * determine current savings due to compression. */
+      if (block_left < estimated_size && estimated_addition > 2000)
+        {
+          svn_stringbuf_t *serialized
+            = svn_stringbuf_create_ensure(get_block_left(context), iterpool);
+          svn_stream_t *temp_stream
+            = svn_stream_from_stringbuf(serialized, iterpool);
+
+          SVN_ERR(svn_fs_fs__write_changes_container(temp_stream, container,
+                                                     iterpool));
+          SVN_ERR(svn_stream_close(temp_stream));
+
+          block_left = get_block_left(context) - serialized->len;
+          estimated_addition = 0;
+        }
 
-      if ((block_left < entry->size) && sub_items->nelts)
+      if ((block_left < estimated_size) && sub_items->nelts)
         {
           SVN_ERR(write_changes_container(context, container, sub_items,
                                           new_entries, iterpool));
@@ -1298,10 +1319,11 @@ write_changes_containers(pack_context_t 
           svn_pool_clear(container_pool);
           container = svn_fs_fs__changes_create(1000, container_pool);
           block_left = get_block_left(context);
+          estimated_addition = 0;
         }
 
       /* still enough space in current block? */
-      if (block_left < entry->size)
+      if (block_left < estimated_size)
         {
           SVN_ERR(auto_pad_block(context, iterpool));
           block_left = get_block_left(context);
@@ -1314,6 +1336,8 @@ write_changes_containers(pack_context_t 
       SVN_ERR(svn_fs_fs__read_changes(&changes, temp_stream, iterpool));
       SVN_ERR(svn_fs_fs__changes_append_list(&list_index, container, changes));
       SVN_ERR_ASSERT(list_index == sub_items->nelts);
+      block_left -= estimated_size;
+      estimated_addition += estimated_size;
       
       APR_ARRAY_PUSH(sub_items, svn_fs_fs__id_part_t) = entry->items[0];