High memory usage when using svn_fs_make_file()

[Lars Jessen <LJ...@mikrov.dk>]

Hi,

I'm trying to use the the svn_fs module to build a database of 10000
files, but it seems that every call to svn_fs_make_file() eats about
300+kb of memory. The
following code illustrates the problem:

#include <apr_lib.h>
#include <svn_cmdline.h>
#include <svn_fs.h>
#include <svn_pools.h>
#include <iostream>
#include <strstream>

void CheckError(svn_error_t* error){
    if(error == NULL)
        return;
    std::cerr << "Error: " << error->message << std::endl;
    exit(-1);
}

int main(){
    const char* repositoryPath = "d:/svntest";

    svn_cmdline_init("svntest", stderr);

    apr_allocator_t *allocator;
    apr_allocator_create(&allocator);
    apr_allocator_max_free_set(allocator,
SVN_ALLOCATOR_RECOMMENDED_MAX_FREE);    

    apr_pool_t *pool = svn_pool_create_ex(NULL, allocator);
    apr_allocator_owner_set(allocator, pool);
    if(NULL == pool){
        std::cerr << "Error initializing pool" << std::endl;
        return -1;
    }

    svn_error_t* error;

    error = svn_fs_initialize(pool);
    CheckError(error);

    svn_fs_t* fs;

    error = svn_fs_open(&fs, repositoryPath, NULL, pool);
    if(error){
        apr_hash_t* hash = apr_hash_make(pool);
        apr_hash_set(hash, SVN_FS_CONFIG_FS_TYPE, APR_HASH_KEY_STRING,
SVN_FS_TYPE_BDB);

        error = svn_fs_create(&fs, repositoryPath, NULL, pool);
        CheckError(error);
    }

    // Get revision
    svn_revnum_t youngest_rev;
    error = svn_fs_youngest_rev(&youngest_rev, fs, pool);
    CheckError(error);

    svn_fs_txn_t *txn;

    apr_pool_t *txpool = svn_pool_create_ex(pool, allocator);

    error = svn_fs_begin_txn(&txn, fs, youngest_rev, txpool);
    CheckError(error);

    svn_fs_root_t* root; 
    error = svn_fs_txn_root(&root, txn, txpool);
    CheckError(error);

    std::cout << "creating files" << std::endl;

    for(int i=0;i<1000;i++){
        std::strstream s;
        s << "file" << i << ".txt";
        error = svn_fs_make_file(root, s.str(), txpool);
        CheckError(error);
    }

    std::cout << "commit'ing" << std::endl;

    const char* conflict;
    error = svn_fs_commit_txn(&conflict, &youngest_rev, txn, txpool);
    CheckError(error);
    svn_pool_clear(txpool);
    svn_pool_destroy(txpool);

    std::cout << "ok" << std::endl;

    svn_pool_clear(pool);
    svn_pool_destroy(pool);

    return 0;
}



The above code takes a long time to complete and memory usage peeks at
about 600MB.
What am I doing wrong?
I'm using svn-win32-1.4.6

- Lars

---------------------------------------------------------------------
To unsubscribe, e-mail: users-unsubscribe@subversion.tigris.org
For additional commands, e-mail: users-help@subversion.tigris.org

Re: High memory usage when using svn_fs_make_file()

[Erik Huelsmann <eh...@gmail.com>]

On Jan 21, 2008 1:03 PM, Lars Jessen <LJ...@mikrov.dk> wrote:
> Hi,
>
> I'm trying to use the the svn_fs module to build a database of 10000
> files, but it seems that every call to svn_fs_make_file() eats about
> 300+kb of memory. The
> following code illustrates the problem:
>
> #include <apr_lib.h>
> #include <svn_cmdline.h>
> #include <svn_fs.h>
> #include <svn_pools.h>
> #include <iostream>
> #include <strstream>
>
> void CheckError(svn_error_t* error){
>     if(error == NULL)
>         return;
>     std::cerr << "Error: " << error->message << std::endl;
>     exit(-1);
> }
>
> int main(){
>     const char* repositoryPath = "d:/svntest";
>
>     svn_cmdline_init("svntest", stderr);
>
>     apr_allocator_t *allocator;
>     apr_allocator_create(&allocator);
>     apr_allocator_max_free_set(allocator,
> SVN_ALLOCATOR_RECOMMENDED_MAX_FREE);
>
>     apr_pool_t *pool = svn_pool_create_ex(NULL, allocator);
>     apr_allocator_owner_set(allocator, pool);
>     if(NULL == pool){
>         std::cerr << "Error initializing pool" << std::endl;
>         return -1;
>     }
>
>     svn_error_t* error;
>
>     error = svn_fs_initialize(pool);
>     CheckError(error);
>
>     svn_fs_t* fs;
>
>     error = svn_fs_open(&fs, repositoryPath, NULL, pool);
>     if(error){
>         apr_hash_t* hash = apr_hash_make(pool);
>         apr_hash_set(hash, SVN_FS_CONFIG_FS_TYPE, APR_HASH_KEY_STRING,
> SVN_FS_TYPE_BDB);
>
>         error = svn_fs_create(&fs, repositoryPath, NULL, pool);
>         CheckError(error);
>     }
>
>     // Get revision
>     svn_revnum_t youngest_rev;
>     error = svn_fs_youngest_rev(&youngest_rev, fs, pool);
>     CheckError(error);
>
>     svn_fs_txn_t *txn;
>
>     apr_pool_t *txpool = svn_pool_create_ex(pool, allocator);
>
>     error = svn_fs_begin_txn(&txn, fs, youngest_rev, txpool);
>     CheckError(error);
>
>     svn_fs_root_t* root;
>     error = svn_fs_txn_root(&root, txn, txpool);
>     CheckError(error);
>
>     std::cout << "creating files" << std::endl;
>
>     for(int i=0;i<1000;i++){
>         std::strstream s;
>         s << "file" << i << ".txt";
>         error = svn_fs_make_file(root, s.str(), txpool);
>         CheckError(error);
>     }

If / When you're in a loop, you're supposed to use an iterpool and
clear that on every iteration. That way, the previous loop should be
limited to 300kB memory usage.

>     std::cout << "commit'ing" << std::endl;
>
>     const char* conflict;
>     error = svn_fs_commit_txn(&conflict, &youngest_rev, txn, txpool);
>     CheckError(error);
>     svn_pool_clear(txpool);
>     svn_pool_destroy(txpool);
>
>     std::cout << "ok" << std::endl;
>
>     svn_pool_clear(pool);
>     svn_pool_destroy(pool);
>
>     return 0;
> }
>
>
>
> The above code takes a long time to complete and memory usage peeks at
> about 600MB.
> What am I doing wrong?
> I'm using svn-win32-1.4.6

I hope the above helps.

Bye,

Erik.

---------------------------------------------------------------------
To unsubscribe, e-mail: users-unsubscribe@subversion.tigris.org
For additional commands, e-mail: users-help@subversion.tigris.org