You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@subversion.apache.org by st...@apache.org on 2017/04/16 09:43:17 UTC
svn commit: r1791585 - in /subversion/trunk/subversion/libsvn_fs_x: ./ fs.c
pack.c pack.h rev_file.c rev_file.h tree.c
Author: stefan2
Date: Sun Apr 16 09:43:17 2017
New Revision: 1791585
URL: http://svn.apache.org/viewvc?rev=1791585&view=rev
Log:
Merge packing fixes and improvements from FSFS to FSX.
This merges revisions r1759117 and 1759122 to 1759126.
Text conflicts were mostly due to renamed functions.
Modified:
subversion/trunk/subversion/libsvn_fs_x/ (props changed)
subversion/trunk/subversion/libsvn_fs_x/fs.c
subversion/trunk/subversion/libsvn_fs_x/pack.c
subversion/trunk/subversion/libsvn_fs_x/pack.h
subversion/trunk/subversion/libsvn_fs_x/rev_file.c
subversion/trunk/subversion/libsvn_fs_x/rev_file.h
subversion/trunk/subversion/libsvn_fs_x/tree.c
Propchange: subversion/trunk/subversion/libsvn_fs_x/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Sun Apr 16 09:43:17 2017
@@ -95,5 +95,5 @@
/subversion/branches/verify-at-commit/subversion/libsvn_fs_x:1462039-1462408
/subversion/branches/verify-keep-going/subversion/libsvn_fs_x:1439280-1492639,1546002-1546110
/subversion/branches/wc-collate-path/subversion/libsvn_fs_x:1402685-1480384
-/subversion/trunk/subversion/libsvn_fs_fs:1415133-1596500,1596567,1597414,1597989,1598273,1599140,1600872,1601633,1603485-1603487,1603499,1603605,1604128,1604188,1604413-1604414,1604416-1604417,1604421,1604442,1604700,1604717,1604720,1604726,1604755,1604794,1604802,1604824,1604836,1604844,1604902-1604903,1604911,1604925,1604933,1604947,1605059-1605060,1605064-1605065,1605068,1605071-1605073,1605075,1605123,1605188-1605189,1605191,1605197,1605444,1605633,1606132,1606142,1606144,1606514,1606526,1606528,1606551,1606554,1606564,1606598-1606599,1606656,1606658,1606662,1606744,1606840,1607085,1607572,1612407,1612810,1613339,1613872,1614611,1615348,1615351-1615352,1615356,1616338-1616339,1616613,1617586,1617688,1618138,1618151,1618153,1618226,1618641,1618653,1618662,1619068,1619358,1619413,1619769,1619774,1620602,1620909,1620912,1620928,1620930,1621275,1621635,1622931,1622937,1622942,1622946,1622959-1622960,1622963,1622987,1623007,1623368,1623373,1623377,1623379,1623381,1623398,1623402,162
4011,1624265,1624512,1626246,1626871,1626873,1626886,1627497-1627498,1627502,1627947-1627949,1627966,1628083,1628093,1628158-1628159,1628161,1628392-1628393,1628415,1628427,1628676,1628738,1628762,1628764,1629854-1629855,1629857,1629865,1629873,1629875,1629879,1630067,1630070,1631049-1631051,1631075,1631115,1631171,1631180,1631185-1631186,1631196-1631197,1631239-1631240,1631548,1631550,1631563,1631567,1631588,1631598,1632646,1632776,1632849,1632851-1632853,1632856-1632857,1632868,1632908,1632926,1633232,1633617-1633618,1634872,1634875,1634879-1634880,1634920,1636478,1636483,1636629,1636644,1637184,1637186,1637330,1637358,1637363,1637393,1639319,1639322,1639335,1639348,1639352,1639355,1639358,1639414,1639419,1639426,1639430,1639436,1639440,1639549,1640061-1640062,1640197,1640915,1640966,1641013,1643139,1643233,1645567,1646021,1646712,1646716,1647537,1647540-1647541,1647820,1647905,1648230,1648238,1648241-1648243,1648253,1648272,1648532,1648537-1648539,1648542,1648591,1648612,1649590,
1651567,1652068,1652076,1652441,1652451,1653608,1654932,1654934,1654937,1655635,1655649,1655651,1655664,1656176,1657525,1657972,1657978,1658482,1659212,1659217,1659314,1659509,1662668,1665318,1665854,1665894,1667090,1667101,1667538,1669743,1669746,1669749,1669945,1670139,1670953,1673170,1673197,1673202,1673204,1673445,1673454,1673685,1673689,1673875,1674165,1674341,1674400,1674404,1674631,1674669,1674673,1675396,1676667,1677431,1678149,1678151,1678718,1678725,1679169,1679907,1679920-1679924,1679926,1680347,1680460,1680464,1680476,1680819,1681949,1681966,1681974,1681994,1682008,1682076,1682086,1682093,1682259,1682265,1682739,1682864,1683311,1683330,1683378,1683544,1683553,1684047,1686232,1686542,1686546,1686554,1686557,1687061,1687064,1687070-1687071,1687074,1687078-1687079,1688270,1688425,1692650,1693886,1694489,1694848,1696171,1696185,1696627-1696628,1696630,1696758,1697372,1697381,1697387,1697393,1697403,1697405,1701017,1701053,1702600,1702922,1703069,1703142,1703237,1703240,17052
66,1705638,1705643,1705646,1705724,1705730,1705739,1706612,1706615,1706617,1706619,1706675-1706676,1706679,1706979-1706980,1707308,1707971-1707973,1707986,1707988-1707989,1708004,1709388,1709799,1710017,1710359,1710368,1710370,1711507,1711582,1711672,1712927,1715793,1715947,1716047,1716067,1716784,1716973-1716974,1717332,1717334,1717864,1719269,1719336,1719413,1719730,1720015,1721285,1723715,1723720,1723834,1723839,1725179-1725180,1726004,1726099,1726116,1726897,1726995,1727006-1727007,1727028,1727040,1727707,1727822,1730491,1735916,1736357,1736359,1737355-1737356,1740721-1740722,1741096,1741200,1741206,1741214,1741224,1742540,1745055,1745107,1745852,1746006,1746012,1746026,1756258-1756266,1756364,1756377,1759135,1781655,1781694,1785053,1785737-1785738,1785741,1785754,1786445-1786446
+/subversion/trunk/subversion/libsvn_fs_fs:1415133-1596500,1596567,1597414,1597989,1598273,1599140,1600872,1601633,1603485-1603487,1603499,1603605,1604128,1604188,1604413-1604414,1604416-1604417,1604421,1604442,1604700,1604717,1604720,1604726,1604755,1604794,1604802,1604824,1604836,1604844,1604902-1604903,1604911,1604925,1604933,1604947,1605059-1605060,1605064-1605065,1605068,1605071-1605073,1605075,1605123,1605188-1605189,1605191,1605197,1605444,1605633,1606132,1606142,1606144,1606514,1606526,1606528,1606551,1606554,1606564,1606598-1606599,1606656,1606658,1606662,1606744,1606840,1607085,1607572,1612407,1612810,1613339,1613872,1614611,1615348,1615351-1615352,1615356,1616338-1616339,1616613,1617586,1617688,1618138,1618151,1618153,1618226,1618641,1618653,1618662,1619068,1619358,1619413,1619769,1619774,1620602,1620909,1620912,1620928,1620930,1621275,1621635,1622931,1622937,1622942,1622946,1622959-1622960,1622963,1622987,1623007,1623368,1623373,1623377,1623379,1623381,1623398,1623402,162
4011,1624265,1624512,1626246,1626871,1626873,1626886,1627497-1627498,1627502,1627947-1627949,1627966,1628083,1628093,1628158-1628159,1628161,1628392-1628393,1628415,1628427,1628676,1628738,1628762,1628764,1629854-1629855,1629857,1629865,1629873,1629875,1629879,1630067,1630070,1631049-1631051,1631075,1631115,1631171,1631180,1631185-1631186,1631196-1631197,1631239-1631240,1631548,1631550,1631563,1631567,1631588,1631598,1632646,1632776,1632849,1632851-1632853,1632856-1632857,1632868,1632908,1632926,1633232,1633617-1633618,1634872,1634875,1634879-1634880,1634920,1636478,1636483,1636629,1636644,1637184,1637186,1637330,1637358,1637363,1637393,1639319,1639322,1639335,1639348,1639352,1639355,1639358,1639414,1639419,1639426,1639430,1639436,1639440,1639549,1640061-1640062,1640197,1640915,1640966,1641013,1643139,1643233,1645567,1646021,1646712,1646716,1647537,1647540-1647541,1647820,1647905,1648230,1648238,1648241-1648243,1648253,1648272,1648532,1648537-1648539,1648542,1648591,1648612,1649590,
1651567,1652068,1652076,1652441,1652451,1653608,1654932,1654934,1654937,1655635,1655649,1655651,1655664,1656176,1657525,1657972,1657978,1658482,1659212,1659217,1659314,1659509,1662668,1665318,1665854,1665894,1667090,1667101,1667538,1669743,1669746,1669749,1669945,1670139,1670953,1673170,1673197,1673202,1673204,1673445,1673454,1673685,1673689,1673875,1674165,1674341,1674400,1674404,1674631,1674669,1674673,1675396,1676667,1677431,1678149,1678151,1678718,1678725,1679169,1679907,1679920-1679924,1679926,1680347,1680460,1680464,1680476,1680819,1681949,1681966,1681974,1681994,1682008,1682076,1682086,1682093,1682259,1682265,1682739,1682864,1683311,1683330,1683378,1683544,1683553,1684047,1686232,1686542,1686546,1686554,1686557,1687061,1687064,1687070-1687071,1687074,1687078-1687079,1688270,1688425,1692650,1693886,1694489,1694848,1696171,1696185,1696627-1696628,1696630,1696758,1697372,1697381,1697387,1697393,1697403,1697405,1701017,1701053,1702600,1702922,1703069,1703142,1703237,1703240,17052
66,1705638,1705643,1705646,1705724,1705730,1705739,1706612,1706615,1706617,1706619,1706675-1706676,1706679,1706979-1706980,1707308,1707971-1707973,1707986,1707988-1707989,1708004,1709388,1709799,1710017,1710359,1710368,1710370,1711507,1711582,1711672,1712927,1715793,1715947,1716047,1716067,1716784,1716973-1716974,1717332,1717334,1717864,1719269,1719336,1719413,1719730,1720015,1721285,1723715,1723720,1723834,1723839,1725179-1725180,1726004,1726099,1726116,1726897,1726995,1727006-1727007,1727028,1727040,1727707,1727822,1730491,1735916,1736357,1736359,1737355-1737356,1740721-1740722,1741096,1741200,1741206,1741214,1741224,1742540,1745055,1745107,1745852,1746006,1746012,1746026,1756258-1756266,1756364,1756377,1759117,1759122-1759126,1759135,1781655,1781694,1785053,1785737-1785738,1785741,1785754,1786445-1786446
/subversion/trunk/subversion/libsvn_fs_x:1414756-1509914
Modified: subversion/trunk/subversion/libsvn_fs_x/fs.c
URL: http://svn.apache.org/viewvc/subversion/trunk/subversion/libsvn_fs_x/fs.c?rev=1791585&r1=1791584&r2=1791585&view=diff
==============================================================================
--- subversion/trunk/subversion/libsvn_fs_x/fs.c (original)
+++ subversion/trunk/subversion/libsvn_fs_x/fs.c Sun Apr 16 09:43:17 2017
@@ -508,7 +508,7 @@ x_pack(svn_fs_t *fs,
apr_pool_t *common_pool)
{
SVN_ERR(x_open(fs, path, common_pool_lock, scratch_pool, common_pool));
- return svn_fs_x__pack(fs, notify_func, notify_baton,
+ return svn_fs_x__pack(fs, 0, notify_func, notify_baton,
cancel_func, cancel_baton, scratch_pool);
}
Modified: subversion/trunk/subversion/libsvn_fs_x/pack.c
URL: http://svn.apache.org/viewvc/subversion/trunk/subversion/libsvn_fs_x/pack.c?rev=1791585&r1=1791584&r2=1791585&view=diff
==============================================================================
--- subversion/trunk/subversion/libsvn_fs_x/pack.c (original)
+++ subversion/trunk/subversion/libsvn_fs_x/pack.c Sun Apr 16 09:43:17 2017
@@ -216,7 +216,7 @@ typedef struct pack_context_t
* to NULL that we already processed. */
apr_array_header_t *reps;
- /* array of int, marking for each revision, the which offset their items
+ /* array of int, marking for each revision, at which offset their items
* begin in REPS. Will be filled in phase 2 and be cleared after
* each revision range. */
apr_array_header_t *rev_offsets;
@@ -344,6 +344,7 @@ reset_pack_context(pack_context_t *conte
SVN_ERR(svn_io_file_trunc(context->reps_file, 0, scratch_pool));
svn_pool_clear(context->info_pool);
+ context->paths = svn_prefix_tree__create(context->info_pool);
return SVN_NO_ERROR;
}
@@ -1839,15 +1840,17 @@ append_revision(pack_context_t *context,
apr_pool_t *iterpool = svn_pool_create(scratch_pool);
svn_fs_x__revision_file_t *rev_file;
apr_file_t *file;
- svn_filesize_t revfile_size;
+ svn_filesize_t revdata_size;
- /* Copy all the bits from the rev file to the end of the pack file. */
+ /* Copy all non-index contents the rev file to the end of the pack file. */
SVN_ERR(svn_fs_x__rev_file_init(&rev_file, context->fs, context->start_rev,
scratch_pool));
- SVN_ERR(svn_fs_x__rev_file_get(&file, rev_file));
+ SVN_ERR(svn_fs_x__rev_file_data_size(&revdata_size, rev_file));
- SVN_ERR(svn_io_file_size_get(&revfile_size, file, scratch_pool));
- SVN_ERR(copy_file_data(context, context->pack_file, file, revfile_size,
+ SVN_ERR(svn_fs_x__rev_file_get(&file, rev_file));
+ SVN_ERR(svn_io_file_aligned_seek(file, ffd->block_size, NULL, 0,
+ iterpool));
+ SVN_ERR(copy_file_data(context, context->pack_file, file, revdata_size,
iterpool));
/* mark the start of a new revision */
@@ -1856,7 +1859,7 @@ append_revision(pack_context_t *context,
/* read the phys-to-log index file until we covered the whole rev file.
* That index contains enough info to build both target indexes from it. */
- while (offset < revfile_size)
+ while (offset < revdata_size)
{
/* read one cluster */
int i;
@@ -1878,7 +1881,7 @@ append_revision(pack_context_t *context,
/* process entry while inside the rev file */
offset = entry->offset;
- if (offset < revfile_size)
+ if (offset < revdata_size)
{
/* there should be true containers */
SVN_ERR_ASSERT(entry->item_count == 1);
@@ -1897,7 +1900,7 @@ append_revision(pack_context_t *context,
}
svn_pool_destroy(iterpool);
- context->pack_offset += revfile_size;
+ context->pack_offset += revdata_size;
return SVN_NO_ERROR;
}
@@ -1956,6 +1959,7 @@ pack_log_addressed(svn_fs_t *fs,
if ( APR_ARRAY_IDX(max_ids, i, apr_uint64_t)
<= (apr_uint64_t)max_items - item_count)
{
+ item_count += APR_ARRAY_IDX(max_ids, i, apr_uint64_t);
context.end_rev++;
}
else
@@ -2048,6 +2052,7 @@ pack_rev_shard(svn_fs_t *fs,
/* In the file system at FS_PATH, pack the SHARD in DIR containing exactly
* MAX_FILES_PER_DIR revisions, using SCRATCH_POOL temporary for allocations.
* COMPRESSION_LEVEL and MAX_PACK_SIZE will be ignored in that case.
+ * An attempt will be made to keep memory usage below MAX_MEM.
*
* CANCEL_FUNC and CANCEL_BATON are what you think they are; similarly
* NOTIFY_FUNC and NOTIFY_BATON.
@@ -2062,6 +2067,7 @@ pack_shard(const char *dir,
int max_files_per_dir,
apr_off_t max_pack_size,
int compression_level,
+ apr_size_t max_mem,
svn_fs_pack_notify_t notify_func,
void *notify_baton,
svn_cancel_func_t cancel_func,
@@ -2093,7 +2099,7 @@ pack_shard(const char *dir,
/* pack the revision content */
SVN_ERR(pack_rev_shard(fs, pack_file_dir, shard_path,
- shard, max_files_per_dir, DEFAULT_MAX_MEM, batch,
+ shard, max_files_per_dir, max_mem, batch,
cancel_func, cancel_baton, scratch_pool));
/* pack the revprops in an equivalent way */
@@ -2158,6 +2164,7 @@ get_pack_status(svn_boolean_t *fully_pac
typedef struct pack_baton_t
{
svn_fs_t *fs;
+ apr_size_t max_mem;
svn_fs_pack_notify_t notify_func;
void *notify_baton;
svn_cancel_func_t cancel_func;
@@ -2223,6 +2230,7 @@ pack_body(void *baton,
ffd->compress_packed_revprops
? SVN__COMPRESSION_ZLIB_DEFAULT
: SVN__COMPRESSION_NONE,
+ pb->max_mem,
pb->notify_func, pb->notify_baton,
pb->cancel_func, pb->cancel_baton, iterpool));
}
@@ -2233,6 +2241,7 @@ pack_body(void *baton,
svn_error_t *
svn_fs_x__pack(svn_fs_t *fs,
+ apr_size_t max_mem,
svn_fs_pack_notify_t notify_func,
void *notify_baton,
svn_cancel_func_t cancel_func,
@@ -2262,5 +2271,7 @@ svn_fs_x__pack(svn_fs_t *fs,
pb.notify_baton = notify_baton;
pb.cancel_func = cancel_func;
pb.cancel_baton = cancel_baton;
+ pb.max_mem = max_mem ? max_mem : DEFAULT_MAX_MEM;
+
return svn_fs_x__with_pack_lock(fs, pack_body, &pb, scratch_pool);
}
Modified: subversion/trunk/subversion/libsvn_fs_x/pack.h
URL: http://svn.apache.org/viewvc/subversion/trunk/subversion/libsvn_fs_x/pack.h?rev=1791585&r1=1791584&r2=1791585&view=diff
==============================================================================
--- subversion/trunk/subversion/libsvn_fs_x/pack.h (original)
+++ subversion/trunk/subversion/libsvn_fs_x/pack.h Sun Apr 16 09:43:17 2017
@@ -26,13 +26,19 @@
#include "fs.h"
/* Possibly pack the repository at PATH. This just take full shards, and
- combines all the revision files into a single one, with a manifest header.
+ combines all the revision files into a single one, with a manifest header
+ when required by the repository format.
+
+ MAX_MEM limits the size of in-memory data structures needed for reordering
+ items. 0 means use the built-in default.
+
Use optional CANCEL_FUNC/CANCEL_BATON for cancellation support.
Use SCRATCH_POOL for temporary allocations.
Existing filesystem references need not change. */
svn_error_t *
svn_fs_x__pack(svn_fs_t *fs,
+ apr_size_t max_mem,
svn_fs_pack_notify_t notify_func,
void *notify_baton,
svn_cancel_func_t cancel_func,
Modified: subversion/trunk/subversion/libsvn_fs_x/rev_file.c
URL: http://svn.apache.org/viewvc/subversion/trunk/subversion/libsvn_fs_x/rev_file.c?rev=1791585&r1=1791584&r2=1791585&view=diff
==============================================================================
--- subversion/trunk/subversion/libsvn_fs_x/rev_file.c (original)
+++ subversion/trunk/subversion/libsvn_fs_x/rev_file.c Sun Apr 16 09:43:17 2017
@@ -474,6 +474,16 @@ svn_fs_x__rev_file_p2l_info(svn_fs_x__in
}
svn_error_t *
+svn_fs_x__rev_file_data_size(svn_filesize_t *size,
+ svn_fs_x__revision_file_t *file)
+{
+ SVN_ERR(auto_read_footer(file));
+ *size = file->l2p_info.start;
+
+ return SVN_NO_ERROR;
+}
+
+svn_error_t *
svn_fs_x__rev_file_seek(svn_fs_x__revision_file_t *file,
apr_off_t *buffer_start,
apr_off_t offset)
Modified: subversion/trunk/subversion/libsvn_fs_x/rev_file.h
URL: http://svn.apache.org/viewvc/subversion/trunk/subversion/libsvn_fs_x/rev_file.h?rev=1791585&r1=1791584&r2=1791585&view=diff
==============================================================================
--- subversion/trunk/subversion/libsvn_fs_x/rev_file.h (original)
+++ subversion/trunk/subversion/libsvn_fs_x/rev_file.h Sun Apr 16 09:43:17 2017
@@ -166,6 +166,12 @@ svn_error_t *
svn_fs_x__rev_file_p2l_info(svn_fs_x__index_info_t *info,
svn_fs_x__revision_file_t *file);
+/* Set *SIZE to the length of the revision data in FILE.
+ */
+svn_error_t *
+svn_fs_x__rev_file_data_size(svn_filesize_t *size,
+ svn_fs_x__revision_file_t *file);
+
/* File manipulation. */
/* Convenience wrapper around svn_io_file_aligned_seek. */
Modified: subversion/trunk/subversion/libsvn_fs_x/tree.c
URL: http://svn.apache.org/viewvc/subversion/trunk/subversion/libsvn_fs_x/tree.c?rev=1791585&r1=1791584&r2=1791585&view=diff
==============================================================================
--- subversion/trunk/subversion/libsvn_fs_x/tree.c (original)
+++ subversion/trunk/subversion/libsvn_fs_x/tree.c Sun Apr 16 09:43:17 2017
@@ -1251,7 +1251,7 @@ svn_fs_x__commit_txn(const char **confli
if (ffd->pack_after_commit)
{
- SVN_ERR(svn_fs_x__pack(fs, NULL, NULL, NULL, NULL, pool));
+ SVN_ERR(svn_fs_x__pack(fs, 0, NULL, NULL, NULL, NULL, pool));
}
return SVN_NO_ERROR;