You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@jackrabbit.apache.org by da...@apache.org on 2015/08/05 22:07:05 UTC
svn commit: r1694315 [3/6] - in /jackrabbit/site/live/oak/docs: ./
architecture/ coldstandby/ nodestore/ nodestore/segment/ oak-mongo-js/
oak-mongo-js/scripts/ oak-mongo-js/scripts/prettify/ oak-mongo-js/styles/
oak_api/ plugins/ query/ security/ secur...
Added: jackrabbit/site/live/oak/docs/nodestore/segment/tar.html
URL: http://svn.apache.org/viewvc/jackrabbit/site/live/oak/docs/nodestore/segment/tar.html?rev=1694315&view=auto
==============================================================================
--- jackrabbit/site/live/oak/docs/nodestore/segment/tar.html (added)
+++ jackrabbit/site/live/oak/docs/nodestore/segment/tar.html Wed Aug 5 20:07:04 2015
@@ -0,0 +1,669 @@
+<!DOCTYPE html>
+<!--
+ | Generated by Apache Maven Doxia at 2015-08-05
+ | Rendered using Apache Maven Fluido Skin 1.3.0
+-->
+<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
+ <head>
+ <meta charset="UTF-8" />
+ <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+ <meta name="Date-Revision-yyyymmdd" content="20150805" />
+ <meta http-equiv="Content-Language" content="en" />
+ <title>Jackrabbit Oak - Structure of TAR files</title>
+ <link rel="stylesheet" href="../../css/apache-maven-fluido-1.3.0.min.css" />
+ <link rel="stylesheet" href="../../css/site.css" />
+ <link rel="stylesheet" href="../../css/print.css" media="print" />
+
+
+ <script type="text/javascript" src="../../js/apache-maven-fluido-1.3.0.min.js"></script>
+
+
+ </head>
+ <body class="topBarEnabled">
+
+
+
+
+
+
+ <a href="http://github.com/apache/jackrabbit-oak">
+ <img style="position: absolute; top: 0; right: 0; border: 0; z-index: 10000;"
+ src="https://s3.amazonaws.com/github/ribbons/forkme_right_red_aa0000.png"
+ alt="Fork me on GitHub">
+ </a>
+
+
+
+
+
+ <div id="topbar" class="navbar navbar-fixed-top ">
+ <div class="navbar-inner">
+ <div class="container-fluid">
+ <a data-target=".nav-collapse" data-toggle="collapse" class="btn btn-navbar">
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ </a>
+
+ <a class="brand" href="../../" title="Oak logo">
+
+
+ <img src="../../oak_logo.png" alt="Oak logo" />
+
+ </a>
+
+ <ul class="nav">
+ <li class="dropdown">
+ <a href="#" class="dropdown-toggle" data-toggle="dropdown">Overview <b class="caret"></b></a>
+ <ul class="dropdown-menu">
+
+ <li> <a href="../../index.html" title="Jackrabbit Oak">Jackrabbit Oak</a>
+</li>
+
+ <li> <a href="../../license.html" title="License">License</a>
+</li>
+
+ <li> <a href="../../downloads.html" title="Downloads">Downloads</a>
+</li>
+ </ul>
+ </li>
+ <li class="dropdown">
+ <a href="#" class="dropdown-toggle" data-toggle="dropdown">Concepts and Architecture <b class="caret"></b></a>
+ <ul class="dropdown-menu">
+
+ <li> <a href="../../architecture/overview.html" title="Overview">Overview</a>
+</li>
+
+ <li> <a href="../../architecture/nodestate.html" title="The Node State Model">The Node State Model</a>
+</li>
+ </ul>
+ </li>
+ <li class="dropdown">
+ <a href="#" class="dropdown-toggle" data-toggle="dropdown">Main APIs <b class="caret"></b></a>
+ <ul class="dropdown-menu">
+
+ <li> <a href="http://www.day.com/specs/jcr/2.0/index.html" title="JCR API">JCR API</a>
+</li>
+
+ <li> <a href="../../oak_api/overview.html" title="Oak API">Oak API</a>
+</li>
+ </ul>
+ </li>
+ <li class="dropdown">
+ <a href="#" class="dropdown-toggle" data-toggle="dropdown">Features and Plugins <b class="caret"></b></a>
+ <ul class="dropdown-menu">
+
+ <li> <a href="../../nodestore/overview.html" title="Node Storage">Node Storage</a>
+</li>
+
+ <li> <a href="../../plugins/blobstore.html" title="Blob Storage">Blob Storage</a>
+</li>
+
+ <li> <a href="../../nodestore/documentmk.html" title="DocumentNodeStore">DocumentNodeStore</a>
+</li>
+
+ <li> <a href="../../nodestore/segment/overview.html" title="Segment Node Store">Segment Node Store</a>
+</li>
+
+ <li> <a href="../../query/query.html" title="Query">Query</a>
+</li>
+
+ <li> <a href="../../query/lucene.html" title="Index - Lucene">Index - Lucene</a>
+</li>
+
+ <li> <a href="../../query/solr.html" title="Index - Solr">Index - Solr</a>
+</li>
+
+ <li> <a href="../../security/overview.html" title="Security">Security</a>
+</li>
+
+ <li> <a href="../../clustering.html" title="Clustering">Clustering</a>
+</li>
+
+ <li> <a href="../../nodestore/persistent-cache.html" title="Persistent Cache">Persistent Cache</a>
+</li>
+ </ul>
+ </li>
+ <li class="dropdown">
+ <a href="#" class="dropdown-toggle" data-toggle="dropdown">Using Oak <b class="caret"></b></a>
+ <ul class="dropdown-menu">
+
+ <li> <a href="../../use_getting_started.html" title="Getting Started">Getting Started</a>
+</li>
+
+ <li> <a href="../../construct.html" title="Repository Construction">Repository Construction</a>
+</li>
+
+ <li> <a href="../../osgi_config.html" title="Configuring Oak">Configuring Oak</a>
+</li>
+
+ <li> <a href="../../command_line.html" title="Command Line Tools">Command Line Tools</a>
+</li>
+
+ <li> <a href="../../differences.html" title="Differences to Jackrabbit 2">Differences to Jackrabbit 2</a>
+</li>
+
+ <li> <a href="../../known_issues.html" title="Known Issues">Known Issues</a>
+</li>
+
+ <li> <a href="../../dos_and_donts.html" title="Dos and Don'ts">Dos and Don'ts</a>
+</li>
+
+ <li> <a href="../../coldstandby/coldstandby.html" title="Cold Standby">Cold Standby</a>
+</li>
+
+ <li> <a href="../../FAQ.html" title="FAQ">FAQ</a>
+</li>
+ </ul>
+ </li>
+ <li class="dropdown">
+ <a href="#" class="dropdown-toggle" data-toggle="dropdown">Developing Oak <b class="caret"></b></a>
+ <ul class="dropdown-menu">
+
+ <li> <a href="../../dev_getting_started.html" title="Getting Started">Getting Started</a>
+</li>
+
+ <li> <a href="../../participating.html" title="Participating">Participating</a>
+</li>
+
+ <li> <a href="../../developing-with-git.html" title="Developing with Git">Developing with Git</a>
+</li>
+
+ <li> <a href="../../attribution.html" title="Attribution">Attribution</a>
+</li>
+ </ul>
+ </li>
+ <li class="dropdown">
+ <a href="#" class="dropdown-toggle" data-toggle="dropdown">Links <b class="caret"></b></a>
+ <ul class="dropdown-menu">
+
+ <li> <a href="http://jackrabbit.apache.org/oak" title="Apache Jackrabbit Oak">Apache Jackrabbit Oak</a>
+</li>
+
+ <li> <a href="http://jackrabbit.apache.org/" title="Apache Jackrabbit">Apache Jackrabbit</a>
+</li>
+ </ul>
+ </li>
+ </ul>
+
+
+
+
+ </div>
+
+ </div>
+ </div>
+ </div>
+
+ <div class="container-fluid">
+ <div id="banner">
+ <div class="pull-left">
+ <div id="bannerLeft">
+ <h2>Oak Documentation</h2>
+ </div>
+ </div>
+ <div class="pull-right"> </div>
+ <div class="clear"><hr/></div>
+ </div>
+
+ <div id="breadcrumbs">
+ <ul class="breadcrumb">
+
+
+ <li id="publishDate">Last Published: 2015-08-05</li>
+ <li class="divider">|</li> <li id="projectVersion">Version: 1.4-SNAPSHOT</li>
+
+
+
+
+ </ul>
+ </div>
+
+
+ <div class="row-fluid">
+ <div id="leftColumn" class="span3">
+ <div class="well sidebar-nav">
+
+
+ <ul class="nav nav-list">
+ <li class="nav-header">Overview</li>
+
+ <li>
+
+ <a href="../../index.html" title="Jackrabbit Oak">
+ <i class="none"></i>
+ Jackrabbit Oak</a>
+ </li>
+
+ <li>
+
+ <a href="../../license.html" title="License">
+ <i class="none"></i>
+ License</a>
+ </li>
+
+ <li>
+
+ <a href="../../downloads.html" title="Downloads">
+ <i class="none"></i>
+ Downloads</a>
+ </li>
+ <li class="nav-header">Concepts and Architecture</li>
+
+ <li>
+
+ <a href="../../architecture/overview.html" title="Overview">
+ <i class="none"></i>
+ Overview</a>
+ </li>
+
+ <li>
+
+ <a href="../../architecture/nodestate.html" title="The Node State Model">
+ <i class="none"></i>
+ The Node State Model</a>
+ </li>
+ <li class="nav-header">Main APIs</li>
+
+ <li>
+
+ <a href="http://www.day.com/specs/jcr/2.0/index.html" class="externalLink" title="JCR API">
+ <i class="none"></i>
+ JCR API</a>
+ </li>
+
+ <li>
+
+ <a href="../../oak_api/overview.html" title="Oak API">
+ <i class="none"></i>
+ Oak API</a>
+ </li>
+ <li class="nav-header">Features and Plugins</li>
+
+ <li>
+
+ <a href="../../nodestore/overview.html" title="Node Storage">
+ <i class="none"></i>
+ Node Storage</a>
+ </li>
+
+ <li>
+
+ <a href="../../plugins/blobstore.html" title="Blob Storage">
+ <i class="none"></i>
+ Blob Storage</a>
+ </li>
+
+ <li>
+
+ <a href="../../nodestore/documentmk.html" title="DocumentNodeStore">
+ <i class="none"></i>
+ DocumentNodeStore</a>
+ </li>
+
+ <li>
+
+ <a href="../../nodestore/segment/overview.html" title="Segment Node Store">
+ <i class="none"></i>
+ Segment Node Store</a>
+ </li>
+
+ <li>
+
+ <a href="../../query/query.html" title="Query">
+ <i class="none"></i>
+ Query</a>
+ </li>
+
+ <li>
+
+ <a href="../../query/lucene.html" title="Index - Lucene">
+ <i class="none"></i>
+ Index - Lucene</a>
+ </li>
+
+ <li>
+
+ <a href="../../query/solr.html" title="Index - Solr">
+ <i class="none"></i>
+ Index - Solr</a>
+ </li>
+
+ <li>
+
+ <a href="../../security/overview.html" title="Security">
+ <i class="none"></i>
+ Security</a>
+ </li>
+
+ <li>
+
+ <a href="../../clustering.html" title="Clustering">
+ <i class="none"></i>
+ Clustering</a>
+ </li>
+
+ <li>
+
+ <a href="../../nodestore/persistent-cache.html" title="Persistent Cache">
+ <i class="none"></i>
+ Persistent Cache</a>
+ </li>
+ <li class="nav-header">Using Oak</li>
+
+ <li>
+
+ <a href="../../use_getting_started.html" title="Getting Started">
+ <i class="none"></i>
+ Getting Started</a>
+ </li>
+
+ <li>
+
+ <a href="../../construct.html" title="Repository Construction">
+ <i class="none"></i>
+ Repository Construction</a>
+ </li>
+
+ <li>
+
+ <a href="../../osgi_config.html" title="Configuring Oak">
+ <i class="none"></i>
+ Configuring Oak</a>
+ </li>
+
+ <li>
+
+ <a href="../../command_line.html" title="Command Line Tools">
+ <i class="none"></i>
+ Command Line Tools</a>
+ </li>
+
+ <li>
+
+ <a href="../../differences.html" title="Differences to Jackrabbit 2">
+ <i class="none"></i>
+ Differences to Jackrabbit 2</a>
+ </li>
+
+ <li>
+
+ <a href="../../known_issues.html" title="Known Issues">
+ <i class="none"></i>
+ Known Issues</a>
+ </li>
+
+ <li>
+
+ <a href="../../dos_and_donts.html" title="Dos and Don'ts">
+ <i class="none"></i>
+ Dos and Don'ts</a>
+ </li>
+
+ <li>
+
+ <a href="../../coldstandby/coldstandby.html" title="Cold Standby">
+ <i class="none"></i>
+ Cold Standby</a>
+ </li>
+
+ <li>
+
+ <a href="../../FAQ.html" title="FAQ">
+ <i class="none"></i>
+ FAQ</a>
+ </li>
+ <li class="nav-header">Developing Oak</li>
+
+ <li>
+
+ <a href="../../dev_getting_started.html" title="Getting Started">
+ <i class="none"></i>
+ Getting Started</a>
+ </li>
+
+ <li>
+
+ <a href="../../participating.html" title="Participating">
+ <i class="none"></i>
+ Participating</a>
+ </li>
+
+ <li>
+
+ <a href="../../developing-with-git.html" title="Developing with Git">
+ <i class="none"></i>
+ Developing with Git</a>
+ </li>
+
+ <li>
+
+ <a href="../../attribution.html" title="Attribution">
+ <i class="none"></i>
+ Attribution</a>
+ </li>
+ <li class="nav-header">Links</li>
+
+ <li>
+
+ <a href="http://jackrabbit.apache.org/oak" class="externalLink" title="Apache Jackrabbit Oak">
+ <i class="none"></i>
+ Apache Jackrabbit Oak</a>
+ </li>
+
+ <li>
+
+ <a href="http://jackrabbit.apache.org/" class="externalLink" title="Apache Jackrabbit">
+ <i class="none"></i>
+ Apache Jackrabbit</a>
+ </li>
+ </ul>
+
+
+
+ <hr class="divider" />
+
+ <div id="poweredBy">
+
+ <script type="text/javascript" src="https://apis.google.com/js/plusone.js"></script>
+
+
+ <div class="g-plusone" data-href="http://jackrabbit.apache.org/oak/docs/" data-size="tall" ></div>
+
+ <div class="clear"></div>
+ <div class="clear"></div>
+ <div class="clear"></div>
+ <a href="http://maven.apache.org/" title="Built by Maven" class="poweredBy">
+ <img class="builtBy" alt="Built by Maven" src="../../images/logos/maven-feather.png" />
+ </a>
+ </div>
+ </div>
+ </div>
+
+
+ <div id="bodyColumn" class="span9" >
+
+ <!-- Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. --><h1>Structure of TAR files</h1>
+<p>Here is described the phisical layout of a TAR file as used by Apache Oak. First, a brief introduction of the TAR format is given. Next, more details are provided about the low level information that are written in TAR entries. Finally, it’s described how Oak saves a graph data structure inside the TAR file and how this representation is optimized for fast retrieval.</p>
+<div class="section">
+<h2>Organization of a TAR file<a name="Organization_of_a_TAR_file"></a></h2>
+<p>Phisically speaking, a TAR file is a linear sequence of blocks. A TAR file is terminated by two blocks containing zero bytes. Every block has a size of 512 bytes.</p>
+<p>Logically speaking, a TAR file is a linear sequence of entries. Every entry is represented by two or more blocks. The first block always contains the entry header. Subsequent blocks store the content of the file.</p>
+<p>The entry header is composed of the following fields:</p>
+
+<ul>
+
+<li>
+<p>file name (100 bytes) - name of the file stored in this entry.</p></li>
+
+<li>
+<p>file mode (8 bytes) - string representation of the octal file mode.</p></li>
+
+<li>
+<p>owner’s numeric ID (8 bytes) - string representation of the user ID of the owner of the file.</p></li>
+
+<li>
+<p>group’s numeric ID (8 bytes) - string representation of the group ID of the owner of the file.</p></li>
+
+<li>
+<p>file size (12 bytes) - string representation of the octal size of the file.</p></li>
+
+<li>
+<p>last modification time (12 bytes) - string representation of the octal time stamp when the file was last modified.</p></li>
+
+<li>
+<p>checksum (8 bytes) - checksum for the header data.</p></li>
+
+<li>
+<p>file type (1 byte) - type of the file stored in the entry. This field specifies if the file is a regular file, a hard link or a symbolic link.</p></li>
+
+<li>
+<p>name of linked file (1 byte) - in case the file stored in the entry is a link, this field stores the name of the file pointed to by the link.</p></li>
+</ul></div>
+<div class="section">
+<h2>The TAR file as used by Oak<a name="The_TAR_file_as_used_by_Oak"></a></h2>
+<p>Some fields are not used by Oak. In particular, Oak sets the file mode, the owner’s numeric ID, the group’s numeric ID, the checksum, and the name of linked file to uninteresting values. The only meaningful values assigned to the fields of the entry values are:</p>
+
+<ul>
+
+<li>
+<p>file name: the name of the data file. There are different data files used by Oak. They are described below.</p></li>
+
+<li>
+<p>file size: the size of the data file. The value assigned to this field is trivially computed from the amount of information stored in the data file.</p></li>
+
+<li>
+<p>last modification time: the time stamp when the entry was written.</p></li>
+</ul>
+<p>There are three kind of files stored in a TAR file:</p>
+
+<ul>
+
+<li>
+<p>segments: this type of file contains data about a segment in the segment store. This kind of file has a file name in the form <tt>UUID.CRC2</tt>, where <tt>UUID</tt> is a 128 bit UUID represented as an hexadecimal string and <tt>CRC2</tt> is a zer- padded numeric string representing the CRC2 checksum of the raw segment data.</p></li>
+
+<li>
+<p>graph: this file has a name ending in <tt>.gph</tt> and contains a representaion of a graph. The graph is represented as an adjacency list of UUIDs.</p></li>
+
+<li>
+<p>index: this file has a name ending in <tt>.idx</tt> and contains a sorted list of every segment contained in the TAR file.</p></li>
+</ul>
+<p>The layout of the TAR file used by Oak is engineered for perfomance of read operations. In particular, the most important information is stored in the bottom entries. Reading the entries from the bottom of the file, you encounter first the index, then the graph, then segment files. The idea is that the index must be read first, because it provides a fast tool to locate segments in the rest of the file. Next comes the graph, that describes how segments relate to each other. Last come the segments, whose relative order can be ignored.</p>
+<p>At the same time, the layout of the TAR file allows fast append-only operations when writing. Since the relative order of segment files is not important, segment entries can be written in a first come, first served basis. The index at the end of the file will provide a fast way to access them even if they are scattered around the file.</p></div>
+<div class="section">
+<h2>Segment files<a name="Segment_files"></a></h2>
+<p>Segment files contain raw data about a segment. Even if there are multiple kinds of segments, TAR file only distinguishes between data and non-data segments. A non-data segment is always saved as-is in the TAR file, without further processing. A data segment, instead, is inspected to extract references to other segments.</p>
+<p>A data segment can contain at most 255 references to other segments. These references are simply stored as a list of UUIDs. The referenced segments can be stored inside the current TAR file or outside of it. In the first case, the referenced segment can be found by inspecting the index. In the second case, an external agent is responsible to find the segment in another TAR file.</p>
+<p>The list of segments referenced by a data segment will end up in the graph file. To speed up the process of locating a segment in the list of referenced segment, this list is maintained ordered.</p></div>
+<div class="section">
+<h2>Graph files<a name="Graph_files"></a></h2>
+<p>The graph file represents the relationships between segments stored inside or outside the TAR file. The graph is stored as an adjacency list of UUID, where each UUID represents a segment.</p>
+<p>The format of the graph file is optimized for reading. The graph file is stored in reverse order to maintain the most important information at the end of the file. This strategy is inline with the overall layout of the entries in the TAR file.</p>
+<p>The content of the graph file is divided in three main parts: a graph header, a graph adjacency list and a vertex mapping table.</p>
+<p>The graph header contains the following fields:</p>
+
+<ul>
+
+<li>
+<p>a magic number (4 bytes): identifies the beginning of a graph file.</p></li>
+
+<li>
+<p>size of the graph adjacency list (4 bytes): number of bytes occupied by the graph adjacency list.</p></li>
+
+<li>
+<p>size of the vertex mapping table (4 bytes): number of bytes occupied by the vertex mapping table.</p></li>
+
+<li>
+<p>checksum (4 bytes): a CRC2 checksum of the content of the graph file.</p></li>
+</ul>
+<p>Immediately after the graph header, the graph adjacency list is stored. In the list, each vertex is represented by an integer. Each integer represents an index in the vertex mapping table. For each vertex stored in the adjacency list, the following information are written:</p>
+
+<ul>
+
+<li>
+<p>the integer representing the current vertex.</p></li>
+
+<li>
+<p>zero or more integers for each vertex referenced by the current one.</p></li>
+
+<li>
+<p>a sentinel value representing the list of adjacent vertices (-1).</p></li>
+</ul>
+<p>At the end, the vertex mapping table is stored. This table is just an ordered list of UUIDs. The integers used in the graph adjacency list can be used as index in the vertext mapping table to read the UUID of the corresponding segment. This is a space optimization. Since UUIDs can be repeated more than once in the adjacency list, it make sense to replace them with cheaper placeholders. A UUID is 128 bit long, while an integer just 4.</p></div>
+<div class="section">
+<h2>Index files<a name="Index_files"></a></h2>
+<p>The index file is an ordered list of references to the entries contained in the TAR file. The references are ordered by UUID and they point to the position in the file where the entry is stored. Like the graph file, even the index file is stored backwards.</p>
+<p>The index file is divided in two parts. The first is an index header, the second contains the real data about the index.</p>
+<p>The index data contains the following fields:</p>
+
+<ul>
+
+<li>
+<p>a magic number (4 bytes): identifies the beginning of an index file.</p></li>
+
+<li>
+<p>size fo the index (4 bytes): number of bytes occupied by the index data. This size also contains padding bytes that are added to the index to make it align with the TAR block boundary.</p></li>
+
+<li>
+<p>number of entries (4 bytes): how many entries the index contains.</p></li>
+
+<li>
+<p>checksum (4 bytes): a CRC32 checksum of the content of the index file.</p></li>
+</ul>
+<p>After the header, the content of the index starts. For every entry contained in the index, the following information are stored:</p>
+
+<ul>
+
+<li>
+<p>the most significat bits of the UUID (8 bytes)</p></li>
+
+<li>
+<p>the least significat bits of the UUID (8 bytes)</p></li>
+
+<li>
+<p>the offset in the TAR file where the TAR entry containing the segment is located.</p></li>
+
+<li>
+<p>the size of the entry in the TAR file.</p></li>
+</ul>
+<p>Since the entries in the index are sorted by UUID, and since the UUIDs assigned to the entries are uniformly distributed, when searching an entry by its UUID an efficient algorithm called interpolation search can be used. This algorithm is a variation of binary search. While in binary search the search space (in this case, the array of entry) is halved at every iteration, interpolation search exploits the distribution of the keys to remove a portion of the search space that is potentially bigger than the half of it. Interpolation search is a more natural approximation of the way a person searches in a phone book. If the name to search begins with the letter T, in example, it makes no sense to open the phone book at the half. It is way more efficient, instead, to open the phone book close to the bottom quarter, since names starting with the letter T are more likely to be distributed in that part of the phone book.</p></div>
+ </div>
+ </div>
+ </div>
+
+ <hr/>
+
+ <footer>
+ <div class="container-fluid">
+ <div class="row span12">Copyright © 2012-2015
+ <a href="http://www.apache.org/">The Apache Software Foundation</a>.
+ All Rights Reserved.
+
+ </div>
+
+
+
+
+
+
+
+ <div id="ohloh" class="pull-right">
+ <script type="text/javascript" src="http://www.ohloh.net/p/jackrabbit-oak/widgets/project_thin_badge.js"></script>
+ </div>
+ </div>
+ </footer>
+ </body>
+</html>
\ No newline at end of file
Modified: jackrabbit/site/live/oak/docs/nodestore/segmentmk.html
URL: http://svn.apache.org/viewvc/jackrabbit/site/live/oak/docs/nodestore/segmentmk.html?rev=1694315&r1=1694314&r2=1694315&view=diff
==============================================================================
--- jackrabbit/site/live/oak/docs/nodestore/segmentmk.html (original)
+++ jackrabbit/site/live/oak/docs/nodestore/segmentmk.html Wed Aug 5 20:07:04 2015
@@ -1,13 +1,13 @@
<!DOCTYPE html>
<!--
- | Generated by Apache Maven Doxia at 2015-07-28
+ | Generated by Apache Maven Doxia at 2015-08-05
| Rendered using Apache Maven Fluido Skin 1.3.0
-->
<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <meta name="Date-Revision-yyyymmdd" content="20150728" />
+ <meta name="Date-Revision-yyyymmdd" content="20150805" />
<meta http-equiv="Content-Language" content="en" />
<title>Jackrabbit Oak - Segment Storage Design Overview</title>
<link rel="stylesheet" href="../css/apache-maven-fluido-1.3.0.min.css" />
@@ -102,7 +102,7 @@
<li> <a href="../nodestore/documentmk.html" title="DocumentNodeStore">DocumentNodeStore</a>
</li>
- <li> <a href="../nodestore/segmentmk.html" title="SegmentNodeStore">SegmentNodeStore</a>
+ <li> <a href="../nodestore/segment/overview.html" title="Segment Node Store">Segment Node Store</a>
</li>
<li> <a href="../query/query.html" title="Query">Query</a>
@@ -166,6 +166,9 @@
<li> <a href="../participating.html" title="Participating">Participating</a>
</li>
+ <li> <a href="../developing-with-git.html" title="Developing with Git">Developing with Git</a>
+</li>
+
<li> <a href="../attribution.html" title="Attribution">Attribution</a>
</li>
</ul>
@@ -207,7 +210,7 @@
<ul class="breadcrumb">
- <li id="publishDate">Last Published: 2015-07-28</li>
+ <li id="publishDate">Last Published: 2015-08-05</li>
<li class="divider">|</li> <li id="projectVersion">Version: 1.4-SNAPSHOT</li>
@@ -298,10 +301,12 @@
DocumentNodeStore</a>
</li>
- <li class="active">
+ <li>
- <a href="#"><i class="none"></i>SegmentNodeStore</a>
- </li>
+ <a href="../nodestore/segment/overview.html" title="Segment Node Store">
+ <i class="none"></i>
+ Segment Node Store</a>
+ </li>
<li>
@@ -426,6 +431,13 @@
<li>
+ <a href="../developing-with-git.html" title="Developing with Git">
+ <i class="none"></i>
+ Developing with Git</a>
+ </li>
+
+ <li>
+
<a href="../attribution.html" title="Attribution">
<i class="none"></i>
Attribution</a>
@@ -485,6 +497,7 @@
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. --><h1>Segment Storage Design Overview</h1>
+<p>TODO: merge with <a href="segment/records.html">Records and segments</a>.</p>
<p>The SegmentNodeStore is an Oak storage backend that stores content as various types of <i>records</i> within larger <i>segments</i>. One or more <i>journals</i> are used to track the latest state of the repository. In the Tar implementation only one “root” journal is used.</p>
<p>The SegmentNodeStore was designed from the ground up based on the following key principles:</p>
Added: jackrabbit/site/live/oak/docs/oak-mongo-js/index.html
URL: http://svn.apache.org/viewvc/jackrabbit/site/live/oak/docs/oak-mongo-js/index.html?rev=1694315&view=auto
==============================================================================
--- jackrabbit/site/live/oak/docs/oak-mongo-js/index.html (added)
+++ jackrabbit/site/live/oak/docs/oak-mongo-js/index.html Wed Aug 5 20:07:04 2015
@@ -0,0 +1,63 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+ <meta charset="utf-8">
+ <title>JSDoc: Index</title>
+
+ <script src="scripts/prettify/prettify.js"> </script>
+ <script src="scripts/prettify/lang-css.js"> </script>
+ <!--[if lt IE 9]>
+ <script src="//html5shiv.googlecode.com/svn/trunk/html5.js"></script>
+ <![endif]-->
+ <link type="text/css" rel="stylesheet" href="styles/prettify-tomorrow.css">
+ <link type="text/css" rel="stylesheet" href="styles/jsdoc-default.css">
+</head>
+
+<body>
+
+<div id="main">
+
+ <h1 class="page-title">Index</h1>
+
+
+
+
+
+
+
+ <h3> </h3>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+</div>
+
+<nav>
+ <h2><a href="index.html">Index</a></h2><h3>Namespaces</h3><ul><li><a href="oak.html">oak</a></li></ul>
+</nav>
+
+<br clear="both">
+
+<footer>
+ Documentation generated by <a href="https://github.com/jsdoc3/jsdoc">JSDoc 3.2.3-dev</a> on Wed Aug 05 2015 21:59:07 GMT+0200 (CEST)
+</footer>
+
+<script> prettyPrint(); </script>
+<script src="scripts/linenumber.js"> </script>
+</body>
+</html>
\ No newline at end of file
Added: jackrabbit/site/live/oak/docs/oak-mongo-js/oak-mongo.js.html
URL: http://svn.apache.org/viewvc/jackrabbit/site/live/oak/docs/oak-mongo-js/oak-mongo.js.html?rev=1694315&view=auto
==============================================================================
--- jackrabbit/site/live/oak/docs/oak-mongo-js/oak-mongo.js.html (added)
+++ jackrabbit/site/live/oak/docs/oak-mongo-js/oak-mongo.js.html Wed Aug 5 20:07:04 2015
@@ -0,0 +1,1064 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+ <meta charset="utf-8">
+ <title>JSDoc: Source: oak-mongo.js</title>
+
+ <script src="scripts/prettify/prettify.js"> </script>
+ <script src="scripts/prettify/lang-css.js"> </script>
+ <!--[if lt IE 9]>
+ <script src="//html5shiv.googlecode.com/svn/trunk/html5.js"></script>
+ <![endif]-->
+ <link type="text/css" rel="stylesheet" href="styles/prettify-tomorrow.css">
+ <link type="text/css" rel="stylesheet" href="styles/jsdoc-default.css">
+</head>
+
+<body>
+
+<div id="main">
+
+ <h1 class="page-title">Source: oak-mongo.js</h1>
+
+
+
+
+
+ <section>
+ <article>
+ <pre class="prettyprint source"><code>/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*global print, _, db, Object, ObjectId */
+
+/** @namespace */
+var oak = (function(global){
+ "use strict";
+
+ var api;
+
+ api = function(){
+ print("Oak Mongo Helpers");
+ };
+
+ /**
+ * Collects various stats related to Oak usage of Mongo.
+ *
+ * @memberof oak
+ * @method oak.systemStats
+ * @returns {object} system stats.
+ */
+ api.systemStats = function () {
+ var result = {};
+ result.nodeStats = db.nodes.stats(1024 * 1024);
+ result.blobStats = db.blobs.stats(1024 * 1024);
+ result.clusterStats = db.clusterNodes.find().toArray();
+ result.oakIndexes = db.nodes.find({'_id': /^2\:\/oak\:index\//}).toArray();
+ result.hostInfo = db.hostInfo();
+ result.rootDoc = db.nodes.findOne({'_id' : '0:/'});
+ return result;
+ };
+
+ /**
+ * Collects various stats related to Oak indexes stored under /oak:index.
+ *
+ * @memberof oak
+ * @method indexStats
+ * @returns {Array} index stats.
+ */
+ api.indexStats = function () {
+ var result = [];
+ var totalCount = 0;
+ var totalSize = 0;
+ db.nodes.find({'_id': /^2\:\/oak\:index\//}, {_id: 1}).forEach(function (doc) {
+ var stats = api.getChildStats(api.pathFromId(doc._id));
+ stats.id = doc._id;
+ result.push(stats);
+
+ totalCount += stats.count;
+ totalSize += stats.size;
+ });
+
+ result.push({id: "summary", count: totalCount, size: totalSize, "simple": humanFileSize(totalSize)});
+ return result;
+ };
+
+ /**
+ * Determines the number of child node (including all sub tree)
+ * for a given parent node path. This would be faster compared to
+ * {@link getChildStats} as it does not load the doc and works on
+ * index only.
+ *
+ * Note that there might be some difference between db.nodes.count()
+ * and countChildren('/') as split docs, intermediate docs are not
+ * accounted for
+ *
+ * @memberof oak
+ * @method countChildren
+ * @param {string} path the path of a node.
+ * @returns {number} the number of children, including all descendant nodes.
+ */
+ api.countChildren = function(path){
+ var depth = pathDepth(path);
+ var totalCount = 0;
+ while (true) {
+ var count = db.nodes.count({_id: pathFilter(depth++, path)});
+ if( count === 0){
+ break;
+ }
+ totalCount += count;
+ }
+ return totalCount;
+ };
+
+ /**
+ * Provides stats related to number of child nodes
+ * below given path or total size taken by such nodes.
+ *
+ * @memberof oak
+ * @method getChildStats
+ * @param {string} path the path of a node.
+ * @returns {{count: number, size: number}} statistics about the child nodes
+ * including all descendants.
+ */
+ api.getChildStats = function(path){
+ var count = 0;
+ var size = 0;
+ this.forEachChild(path, function(doc){
+ count++;
+ size += Object.bsonsize(doc);
+ });
+ return {"count" : count, "size" : size, "simple" : humanFileSize(size)};
+ };
+
+ /**
+ * Performs a breadth first traversal for nodes under given path
+ * and invokes the passed function for each child node.
+ *
+ * @memberof oak
+ * @method forEachChild
+ * @param {string} path the path of a node.
+ * @param callable a function to be called for each child node including all
+ * descendant nodes. The MongoDB document is passed as the single
+ * parameter of the function.
+ */
+ api.forEachChild = function(path, callable) {
+ var depth = pathDepth(path);
+ while (true) {
+ var cur = db.nodes.find({_id: pathFilter(depth++, path)});
+ if(!cur.hasNext()){
+ break;
+ }
+ cur.forEach(callable);
+ }
+ };
+
+ /**
+ * Returns the path part of the given id.
+ *
+ * @memberof oak
+ * @method pathFromId
+ * @param {string} id the id of a Document in the nodes collection.
+ * @returns {string} the path derived from the id.
+ */
+ api.pathFromId = function(id) {
+ var index = id.indexOf(':');
+ return id.substring(index + 1);
+ };
+
+ /**
+ * Checks the _lastRev for a given clusterId. The checks starts with the
+ * given path and walks up to the root node.
+ *
+ * @memberof oak
+ * @method checkLastRevs
+ * @param {string} path the path of a node to check
+ * @param {number} clusterId the id of an oak cluster node.
+ * @returns {object} the result of the check.
+ */
+ api.checkLastRevs = function(path, clusterId) {
+ return checkOrFixLastRevs(path, clusterId, true);
+ };
+
+ /**
+ * Fixes the _lastRev for a given clusterId. The fix starts with the
+ * given path and walks up to the root node.
+ *
+ * @memberof oak
+ * @method fixLastRevs
+ * @param {string} path the path of a node to fix
+ * @param {number} clusterId the id of an oak cluster node.
+ * @returns {object} the result of the fix.
+ */
+ api.fixLastRevs = function(path, clusterId) {
+ return checkOrFixLastRevs(path, clusterId, false);
+ };
+
+ /**
+ * Returns statistics about the blobs collection in the current database.
+ * The stats include the combined BSON size of all documents. The time to
+ * run this command therefore heavily depends on the size of the collection.
+ *
+ * @memberof oak
+ * @method blobStats
+ * @returns {object} statistics about the blobs collection.
+ */
+ api.blobStats = function() {
+ var result = {};
+ var stats = db.blobs.stats(1024 * 1024);
+ var bsonSize = 0;
+ db.blobs.find().forEach(function(doc){bsonSize += Object.bsonsize(doc)});
+ result.count = stats.count;
+ result.size = stats.size;
+ result.storageSize = stats.storageSize;
+ result.bsonSize = Math.round(bsonSize / (1024 * 1024));
+ result.indexSize = stats.totalIndexSize;
+ return result;
+ };
+
+ /**
+ * Converts the given Revision String into a more human readable version,
+ * which also prints the date.
+ *
+ * @memberof oak
+ * @method formatRevision
+ * @param {string} rev a revision string.
+ * @returns {string} a human readable string representation of the revision.
+ */
+ api.formatRevision = function(rev) {
+ return new Revision(rev).toReadableString();
+ };
+
+ /**
+ * Removes the complete subtree rooted at the given path.
+ *
+ * @memberof oak
+ * @method removeDescendantsAndSelf
+ * @param {string} path the path of the subtree to remove.
+ */
+ api.removeDescendantsAndSelf = function(path) {
+ var count = 0;
+ var depth = pathDepth(path);
+ var id = depth + ":" + path;
+ // current node at path
+ var result = db.nodes.remove({_id: id});
+ count += result.nRemoved;
+ // might be a long path
+ result = db.nodes.remove(longPathQuery(path));
+ count += result.nRemoved;
+ // descendants
+ var prefix = path + "/";
+ depth++;
+ while (true) {
+ result = db.nodes.remove(longPathFilter(depth, prefix));
+ count += result.nRemoved;
+ result = db.nodes.remove({_id: pathFilter(depth++, prefix)});
+ count += result.nRemoved;
+ if (result.nRemoved == 0) {
+ break;
+ }
+ }
+ // descendants further down the hierarchy with long path
+ while (true) {
+ result = db.nodes.remove(longPathFilter(depth++, prefix));
+ if (result.nRemoved == 0) {
+ break;
+ }
+ count += result.nRemoved;
+ }
+ return {nRemoved : count};
+ };
+
+ /**
+ * List all checkpoints.
+ *
+ * @memberof oak
+ * @method listCheckpoints
+ * @returns {object} all checkpoints
+ */
+ api.listCheckpoints = function() {
+ var result = {};
+ var doc = db.settings.findOne({_id:"checkpoint"});
+ if (doc == null) {
+ print("No checkpoint document found.");
+ return;
+ }
+ var data = doc.data;
+ var r;
+ for (r in data) {
+ var rev = new Revision(r);
+ var exp;
+ if (data[r].charAt(0) == '{') {
+ exp = JSON.parse(data[r])["expires"];
+ } else {
+ exp = data[r];
+ }
+ result[r] = {created:rev.asDate(), expires:new Date(parseInt(exp, 10))};
+ }
+ return result;
+ };
+
+ /**
+ * Removes all checkpoints older than a given Revision.
+ *
+ * @memberof oak
+ * @method removeCheckpointsOlderThan
+ * @param {string} rev checkpoints older than this revision are removed.
+ * @returns {object} the result of the MongoDB update.
+ */
+ api.removeCheckpointsOlderThan = function(rev) {
+ if (rev === undefined) {
+ print("No revision specified");
+ return;
+ }
+ var r = new Revision(rev);
+ var unset = {};
+ var cps = api.listCheckpoints();
+ var x;
+ var num = 0;
+ for (x in cps) {
+ if (r.isNewerThan(new Revision(x))) {
+ unset["data." + x] = "";
+ num++;
+ }
+ }
+ if (num > 0) {
+ var update = {};
+ update["$inc"] = {_modCount: NumberLong(1)};
+ update["$unset"] = unset;
+ return db.settings.update({_id:"checkpoint"}, update);
+ } else {
+ print("No checkpoint older than " + rev);
+ }
+ };
+
+ /**
+ * Removes all collision markers on the document with the given path and
+ * clusterId. This method will only remove collisions when the clusterId
+ * is inactive.
+ *
+ * @memberof oak
+ * @method removeCollisions
+ * @param {string} path the path of a document
+ * @param {number} clusterId collision markers for this clusterId will be removed.
+ * @returns {object} the result of the MongoDB update.
+ */
+ api.removeCollisions = function(path, clusterId) {
+ if (path === undefined) {
+ print("No path specified");
+ return;
+ }
+ if (clusterId === undefined) {
+ print("No clusterId specified");
+ return;
+ }
+ // refuse to remove when clusterId is marked active
+ var clusterNode = db.clusterNodes.findOne({_id: clusterId.toString()});
+ if (clusterNode && clusterNode.state == "ACTIVE") {
+ print("Cluster node with id " + clusterId + " is active!");
+ print("Can only remove collisions for inactive cluster node.");
+ return;
+ }
+
+ var doc = this.findOne(path);
+ if (!doc) {
+ print("No document for path: " + path);
+ return;
+ }
+ var unset = {};
+ var r;
+ var num = 0;
+ for (r in doc._collisions) {
+ if (new Revision(r).getClusterId() == clusterId) {
+ unset["_collisions." + r] = "";
+ num++;
+ }
+ }
+ if (num > 0) {
+ var update = {};
+ update["$inc"] = {_modCount: NumberLong(1)};
+ update["$unset"] = unset;
+ return db.nodes.update({_id: pathDepth(path) + ":" + path}, update);
+ } else {
+ print("No collisions found for clusterId " + clusterId);
+ }
+ };
+
+ /**
+ * Finds the document with the given path.
+ *
+ * @memberof oak
+ * @method findOne
+ * @param {string} path the path of the document.
+ * @returns {object} the document or null if it doesn't exist.
+ */
+ api.findOne = function(path) {
+ if (path === undefined) {
+ return null;
+ }
+ return db.nodes.findOne({_id: pathDepth(path) + ":" + path});
+ };
+
+ /**
+ * Checks the history of previous documents at the given path. Orphaned
+ * references to removed previous documents are counted and listed when
+ * run with verbose set to true.
+ *
+ * @memberof oak
+ * @method checkHistory
+ * @param {string} path the path of the document.
+ * @param {boolean} [verbose=false] if true, the result object will contain a list
+ * of dangling references to previous documents.
+ * @param {boolean} [ignorePathLen=false] whether to ignore a long path and
+ * still try to read it from MongoDB.
+ * @returns {object} the result of the check.
+ */
+ api.checkHistory = function(path, verbose, ignorePathLen) {
+ return checkOrFixHistory(path, false, verbose, ignorePathLen);
+ };
+
+ /**
+ * Lists the descendant documents at a given path.
+ *
+ * @memberof oak
+ * @method listDescendants
+ * @param {string} path list the descendants of the document with this path.
+ */
+ api.listDescendants = function(path) {
+ if (path === undefined) {
+ return null;
+ }
+ var numDescendants = 0;
+ print("Listing descendants for "+path);
+ this.forEachChild(path, function(aChild) {
+ print(api.pathFromId(aChild._id));
+ numDescendants++;
+ });
+ print("Found " + numDescendants + " descendants");
+ };
+
+ /**
+ * Lists the children at a given path.
+ *
+ * @memberof oak
+ * @method listChildren
+ * @param {string} path list the children of the document with this path.
+ */
+ api.listChildren = function(path) {
+ if (path === undefined) {
+ return null;
+ }
+ var numChildren = 0;
+ print("Listing children for "+path);
+ var prefix;
+ if (path == "/") {
+ prefix = path;
+ } else {
+ prefix = path + "/";
+ }
+ db.nodes.find({_id: pathFilter(pathDepth(path) + 1, prefix)}).forEach(function(doc) {
+ print(api.pathFromId(doc._id));
+ numChildren++;
+ });
+ print("Found " + numChildren + " children");
+ };
+
+ /**
+ * Same as checkHistory except it goes through ALL descendants as well!
+ *
+ * @memberof oak
+ * @method checkDeepHistory
+ * @param {string} path the path of the document.
+ * @param {boolean} [verbose=false] if true, the result object will contain a list
+ * of dangling references to previous documents.
+ */
+ api.checkDeepHistory = function(path, verbose) {
+ checkOrFixDeepHistory(path, false, false, verbose);
+ };
+
+ /**
+ * Preparation step which scans through all descendants and prints out
+ * 'fixHistory' for those that need fixing of their 'dangling references'.
+ * <p>
+ * See fixHistory for parameter details.
+ * <p>
+ * Run this command via something as follows:
+ * <p>
+ * mongo <DBNAME> -eval "load('oak-mongo.js'); oak.prepareDeepHistory('/');" > fix.js
+ *
+ * @memberof oak
+ * @method prepareDeepHistory
+ * @param {string} path the path of a document.
+ * @param {boolean} [verbose=false] if true, the result object will contain a list
+ * of dangling references to previous documents.
+ */
+ api.prepareDeepHistory = function(path, verbose) {
+ checkOrFixDeepHistory(path, false, true, verbose);
+ };
+
+ /**
+ * Same as fixHistory except it goes through ALL descendants as well!
+ *
+ * @memberof oak
+ * @method fixDeepHistory
+ * @param {string} path the path of the document.
+ * @param {boolean} [verbose=false] if true, the result object will contain a list
+ * of removed references to previous documents.
+ */
+ api.fixDeepHistory = function(path, verbose) {
+ checkOrFixDeepHistory(path, true, false, verbose);
+ };
+
+ /**
+ * Repairs the history of previous documents at the given path. Orphaned
+ * references to removed previous documents are cleaned up and listed when
+ * run with verbose set to true.
+ *
+ * @memberof oak
+ * @method fixHistory
+ * @param {string} path the path of the document.
+ * @param {boolean} [verbose=false] if true, the result object will contain a list
+ * of removed references to previous documents.
+ * @returns {object} the result of the fix.
+ */
+ api.fixHistory = function(path, verbose) {
+ return checkOrFixHistory(path, true, verbose, true);
+ };
+
+ /**
+ * Returns the commit value entry for the change with the given revision.
+ *
+ * @memberof oak
+ * @method getCommitValue
+ * @param {string} path the path of a document.
+ * @param {string} revision the revision of a change on the document.
+ * @returns {object} the commit entry for the given revision or null if
+ * there is none.
+ */
+ api.getCommitValue = function(path, revision) {
+ var doc = this.findOne(path);
+ if (!doc) {
+ return null;
+ }
+ if (revision === undefined) {
+ print("No revision specified");
+ }
+ // check _revisions
+ var entry = getRevisionEntry(doc, path, revision);
+ if (entry) {
+ return entry;
+ }
+
+ // get commit root
+ entry = getEntry(doc, "_commitRoot", revision);
+ if (!entry) {
+ var prev = findPreviousDocument(path, "_commitRoot", revision);
+ if (prev) {
+ entry = getEntry(prev, "_commitRoot", revision);
+ }
+ }
+ if (!entry) {
+ return null;
+ }
+ var commitRootPath = getCommitRootPath(path, parseInt(entry[revision]));
+ doc = this.findOne(commitRootPath);
+ if (!doc) {
+ return null;
+ }
+ return getRevisionEntry(doc, commitRootPath, revision);
+ };
+
+ /**
+ * Prints mongoexport command to export all documents related to given path.
+ * Related documents refer to all documents in the hierarchy and their split documents.
+ * e.g.
+ * > oak.printMongoExportCommand("/etc", {db: "aem-author"})
+ *
+ * @memberof oak
+ * @method printMongoExportCommand
+ * @param {string} path the path of the document.
+ * @param {object} options pass optional parameters for host, port, db, and filename
+ * @returns {string} command line which can be used to export documents using mongoexport
+ */
+
+ api.printMongoExportCommand = function (path, options) {
+ options = options || {};
+ var host = options.host || "127.0.0.1";
+ var port = options.port || "27017";
+ var db = options.db || "oak";
+ var filename = options.filename || "all-required-nodes.json"
+
+ var query = JSON.stringify(getDocAndHierarchyQuery(path));
+
+ var mongoExportCommand = "mongoexport"
+ + " --host " + host
+ + " --port " + port
+ + " --db " + db
+ + " --collection nodes"
+ + " --out " + filename
+ + " --query '" + query + "'";
+
+ return mongoExportCommand;
+ };
+
+ /**
+ * Prints mongoexport command to export oplog entries around time represented by revision.
+ * e.g.
+ * > oak.printOplogSliceCommand("r14e64620028-0-1", {db: "aem-author"})
+ * Note, this assumed that time on mongo instance is synchronized with time on oak instance. If that's
+ * not the case, then adjust revStr to account for the difference.
+ *
+ * @memberof oak
+ * @method printOplogSliceCommand
+ * @param {string} revStr revision string around which oplog is to be exported.
+ * @param {object} options pass optional parameters for host, port, db, filename, oplogTimeBuffer
+ * @returns {string} command line which can be used to export oplog entries using mongoexport
+ */
+
+ api.printOplogSliceCommand = function (revStr, options) {
+ options = options || {};
+ var host = options.host || "127.0.0.1";
+ var port = options.port || "27017";
+ var db = options.db || "oak";
+ var filename = options.filename || "oplog.json";
+ var oplogTimeBuffer = options.oplogTimeBuffer || 10;
+
+ var rev = new Revision(revStr);
+ var revTimeInSec = rev.asDate().getTime()/1000;
+ var startOplogTime = Math.floor(revTimeInSec - oplogTimeBuffer);
+ var endOplogTime = Math.ceil(revTimeInSec + oplogTimeBuffer);
+
+ var query = '{"ns" : "' + db + '.nodes", "ts": {"$gte": Timestamp(' + startOplogTime
+ + ', 1), "$lte": Timestamp(' + endOplogTime + ', 1)}}';
+
+ var mongoExportCommand = "mongoexport"
+ + " --host " + host
+ + " --port " + port
+ + " --db local"
+ + " --collection oplog.rs"
+ + " --out " + filename
+ + " --query '" + query + "'";
+
+ return mongoExportCommand;
+ };
+
+ //~--------------------------------------------------< internal >
+
+ var checkOrFixDeepHistory = function(path, fix, prepare, verbose) {
+ if (prepare) {
+ // not issuing any header at all
+ } else if (fix) {
+ print("Fixing "+path+" plus all descendants...");
+ } else {
+ print("Checking "+path+" plus all descendants...");
+ }
+ var count = 0;
+ var ignored = 0;
+ var affected = 0;
+ api.forEachChild(path, function(aChild) {
+ var p = api.pathFromId(aChild._id);
+ var result = checkOrFixHistory(p, fix, verbose, true);
+ if (result) {
+ if (prepare) {
+ var numDangling = result.numPrevLinksDangling;
+ if (numDangling!=0) {
+ print("oak.fixHistory('"+p+"');");
+ affected++;
+ }
+ } else if (fix) {
+ var numDangling = result.numPrevLinksRemoved;
+ if (numDangling!=0) {
+ print(" - path: "+p+" removed "+numDangling+" dangling previous revisions");
+ affected++;
+ }
+ } else {
+ var numDangling = result.numPrevLinksDangling;
+ if (numDangling!=0) {
+ print(" - path: "+p+" has "+numDangling+" dangling previous revisions");
+ affected++;
+ }
+ }
+ if (!prepare && (++count%10000==0)) {
+ print("[checked "+count+" so far ("+affected+" affected, "+ignored+" ignored) ...]");
+ }
+ } else {
+ if (!prepare) {
+ print(" - could not handle "+p);
+ }
+ ignored++;
+ }
+ });
+ if (!prepare) {
+ print("Total: "+count+" handled, "+affected+" affected, "+ignored+" ignored (path too long)");
+ print("done.");
+ }
+ };
+
+ var getRevisionEntry = function (doc, path, revision) {
+ var entry = getEntry(doc, "_revisions", revision);
+ if (entry) {
+ return entry;
+ }
+ var prev = findPreviousDocument(path, "_revisions", revision);
+ if (prev) {
+ entry = getEntry(prev, "_revisions", revision);
+ if (entry) {
+ return entry;
+ }
+ }
+ };
+
+ var getCommitRootPath = function(path, depth) {
+ if (depth == 0) {
+ return "/";
+ }
+ var idx = 0;
+ while (depth-- > 0 && idx != -1) {
+ idx = path.indexOf("/", idx + 1);
+ }
+ if (idx == -1) {
+ idx = path.length;
+ }
+ return path.substring(0, idx);
+ };
+
+ var getEntry = function(doc, name, revision) {
+ var result = null;
+ if (doc && doc[name] && doc[name][revision]) {
+ result = {};
+ result[revision] = doc[name][revision];
+ }
+ return result;
+ };
+
+ var findPreviousDocument = function(path, name, revision) {
+ var rev = new Revision(revision);
+ if (path === undefined) {
+ print("No path specified");
+ return;
+ }
+ if (path.length > 165) {
+ print("Path too long");
+ return;
+ }
+ var doc = api.findOne(path);
+ if (!doc) {
+ return null;
+ }
+ var result = null;
+ forEachPrev(doc, function traverse(d, high, low, height) {
+ var highRev = new Revision(high);
+ var lowRev = new Revision(low);
+ if (highRev.getClusterId() != rev.getClusterId()
+ || lowRev.isNewerThan(rev)
+ || rev.isNewerThan(highRev)) {
+ return;
+ }
+
+ var id = prevDocIdFor(path, high, height);
+
+ var prev = db.nodes.findOne({_id: id });
+ if (prev) {
+ if (prev[name] && prev[name][revision]) {
+ result = prev;
+ } else {
+ forEachPrev(prev, traverse);
+ }
+ }
+ });
+ return result;
+ };
+
+ var checkOrFixHistory = function(path, fix, verbose, ignorePathLen) {
+ if (path === undefined) {
+ print("No path specified");
+ return;
+ }
+ if (!ignorePathLen && (path.length > 165)) {
+ print("Path too long");
+ return;
+ }
+
+ var doc = api.findOne(path);
+ if (!doc) {
+ return null;
+ }
+
+ var result = {};
+ result._id = pathDepth(path) + ":" + path;
+ if (verbose) {
+ result.prevDocs = [];
+ if (fix) {
+ result.prevLinksRemoved = [];
+ } else {
+ result.prevLinksDangling = [];
+ }
+ }
+ result.numPrevDocs = 0;
+ if (fix) {
+ result.numPrevLinksRemoved = 0;
+ } else {
+ result.numPrevLinksDangling = 0;
+ }
+
+
+ forEachPrev(doc, function traverse(d, high, low, height) {
+ var id = prevDocIdFor(path, high, height);
+ var prev = db.nodes.findOne({_id: id });
+ if (prev) {
+ if (result.prevDocs) {
+ result.prevDocs.push(high + "/" + height);
+ }
+ result.numPrevDocs++;
+ if (parseInt(height) > 0) {
+ forEachPrev(prev, traverse);
+ }
+ } else if (fix) {
+ if (result.prevLinksRemoved) {
+ result.prevLinksRemoved.push(high + "/" + height);
+ }
+ result.numPrevLinksRemoved++;
+ var update = {};
+ update.$inc = {_modCount : NumberLong(1)};
+ if (d._sdType == 40) { // intermediate split doc type
+ update.$unset = {};
+ update.$unset["_prev." + high] = 1;
+ } else {
+ update.$set = {};
+ update.$set["_stalePrev." + high] = height;
+ }
+ db.nodes.update({_id: d._id}, update);
+ } else {
+ if (result.prevLinksDangling) {
+ result.prevLinksDangling.push(high + "/" + height);
+ }
+ result.numPrevLinksDangling++;
+ }
+ });
+ return result;
+ };
+
+ var forEachPrev = function(doc, callable) {
+ var stalePrev = doc._stalePrev;
+ if (!stalePrev) {
+ stalePrev = {};
+ }
+ var r;
+ for (r in doc._prev) {
+ var value = doc._prev[r];
+ var idx = value.lastIndexOf("/");
+ var height = value.substring(idx + 1);
+ var low = value.substring(0, idx);
+ if (stalePrev[r] == height) {
+ continue;
+ }
+ callable.call(this, doc, r, low, height);
+ }
+ };
+
+ var checkOrFixLastRevs = function(path, clusterId, dryRun) {
+ if (path === undefined) {
+ print("Need at least a path from where to start check/fix.");
+ return;
+ }
+ var result = [];
+ var lastRev;
+ if (path.length == 0 || path.charAt(0) != '/') {
+ return "Not a valid absolute path";
+ }
+ if (clusterId === undefined) {
+ clusterId = 1;
+ }
+ while (true) {
+ var doc = db.nodes.findOne({_id: pathDepth(path) + ":" + path});
+ if (doc) {
+ var revStr = doc._lastRev["r0-0-" + clusterId];
+ if (revStr) {
+ var rev = new Revision(revStr);
+ if (lastRev && lastRev.isNewerThan(rev)) {
+ if (dryRun) {
+ result.push({_id: doc._id, _lastRev: rev.toString(), needsFix: lastRev.toString()});
+ } else {
+ var update = {$set:{}};
+ update.$set["_lastRev.r0-0-" + clusterId] = lastRev.toString();
+ db.nodes.update({_id: doc._id}, update);
+ result.push({_id: doc._id, _lastRev: rev.toString(), fixed: lastRev.toString()});
+ }
+ } else {
+ result.push({_id: doc._id, _lastRev: rev.toString()});
+ lastRev = rev;
+ }
+ }
+ }
+ if (path == "/") {
+ break;
+ }
+ var idx = path.lastIndexOf("/");
+ if (idx == 0) {
+ path = "/";
+ } else {
+ path = path.substring(0, idx);
+ }
+ }
+ return result;
+ };
+
+ var Revision = function(rev) {
+ var dashIdx = rev.indexOf("-");
+ this.rev = rev;
+ this.timestamp = parseInt(rev.substring(1, dashIdx), 16);
+ this.counter = parseInt(rev.substring(dashIdx + 1, rev.indexOf("-", dashIdx + 1)), 16);
+ this.clusterId = parseInt(rev.substring(rev.lastIndexOf("-") + 1), 16);
+ };
+
+ Revision.prototype.toString = function () {
+ return this.rev;
+ };
+
+ Revision.prototype.isNewerThan = function(other) {
+ if (this.timestamp > other.timestamp) {
+ return true;
+ } else if (this.timestamp < other.timestamp) {
+ return false;
+ } else {
+ return this.counter > other.counter;
+ }
+ };
+
+ Revision.prototype.toReadableString = function () {
+ return this.rev + " (" + this.asDate().toString() + ")"
+ };
+
+ Revision.prototype.asDate = function() {
+ return new Date(this.timestamp);
+ };
+
+ Revision.prototype.getClusterId = function() {
+ return this.clusterId;
+ };
+
+ var pathDepth = function(path){
+ if(path === '/'){
+ return 0;
+ }
+ var depth = 0;
+ for(var i = 0; i < path.length; i++){
+ if(path.charAt(i) === '/'){
+ depth++;
+ }
+ }
+ return depth;
+ };
+
+ var prevDocIdFor = function(path, high, height) {
+ var p = "p" + path;
+ if (p.charAt(p.length - 1) != "/") {
+ p += "/";
+ }
+ p += high + "/" + height;
+ return (pathDepth(path) + 2) + ":" + p;
+ };
+
+ var pathFilter = function (depth, prefix){
+ return new RegExp("^"+ depth + ":" + escapeForRegExp(prefix));
+ };
+
+ var longPathFilter = function (depth, prefix) {
+ var filter = {};
+ filter._id = new RegExp("^" + depth + ":h");
+ filter._path = new RegExp("^" + escapeForRegExp(prefix));
+ return filter;
+ };
+
+ var longPathQuery = function (path) {
+ var query = {};
+ query._id = new RegExp("^" + pathDepth(path) + ":h");
+ query._path = path;
+ return query;
+ };
+
+ //http://stackoverflow.com/a/20732091/1035417
+ var humanFileSize = function (size) {
+ var i = Math.floor( Math.log(size) / Math.log(1024) );
+ return ( size / Math.pow(1024, i) ).toFixed(2) * 1 + ' ' + ['B', 'kB', 'MB', 'GB', 'TB'][i];
+ };
+
+ // http://stackoverflow.com/questions/3561493/is-there-a-regexp-escape-function-in-javascript
+ var escapeForRegExp = function(s) {
+ return s.replace(/[-\/\\^$*+?.()|[\]{}]/g, '\\$&');
+ };
+
+ var getDocAndHierarchyQuery = function (path) {
+ var paths = getHierarchyPaths(path);
+
+ var ins = [];
+ var ors = [];
+ paths.forEach(function (path) {
+ ins.push(pathDepth(path) + ':' + path);
+
+ var depth = pathDepth(path);
+ var splitDocRegex = '^' + (depth+2) + ':p' + path + (depth==0?'':'/');
+
+ ors.push({_id : {$regex : splitDocRegex}});
+ });
+
+ ors.push({_id : {$in : ins}});
+
+ return {$or : ors}
+ };
+
+ var getHierarchyPaths = function (path) {
+ var pathElems = path.split("/");
+ var lastPath = "";
+ var paths = ["/"];
+
+ pathElems.forEach(function (pathElem) {
+ //avoid empty path elems like "/".split("/")->["", ""] or "/a".split("/")->["", "a"]
+ if (pathElem != "") {
+ lastPath = lastPath + "/" + pathElem;
+ paths.push(lastPath);
+ }
+ });
+
+ return paths;
+ };
+
+ return api;
+}(this));
+</code></pre>
+ </article>
+ </section>
+
+
+
+
+</div>
+
+<nav>
+ <h2><a href="index.html">Index</a></h2><h3>Namespaces</h3><ul><li><a href="oak.html">oak</a></li></ul>
+</nav>
+
+<br clear="both">
+
+<footer>
+ Documentation generated by <a href="https://github.com/jsdoc3/jsdoc">JSDoc 3.2.3-dev</a> on Wed Aug 05 2015 21:59:07 GMT+0200 (CEST)
+</footer>
+
+<script> prettyPrint(); </script>
+<script src="scripts/linenumber.js"> </script>
+</body>
+</html>
\ No newline at end of file