You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by gi...@apache.org on 2018/09/12 14:54:48 UTC
[47/49] hbase-site git commit: Published site at
3810ba2c6edfc531181ffc9e6c68396a0c2d2027.
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/705d69c4/apidocs/org/apache/hadoop/hbase/snapshot/ExportSnapshot.html
----------------------------------------------------------------------
diff --git a/apidocs/org/apache/hadoop/hbase/snapshot/ExportSnapshot.html b/apidocs/org/apache/hadoop/hbase/snapshot/ExportSnapshot.html
index f1f36a4..fa81722 100644
--- a/apidocs/org/apache/hadoop/hbase/snapshot/ExportSnapshot.html
+++ b/apidocs/org/apache/hadoop/hbase/snapshot/ExportSnapshot.html
@@ -393,7 +393,7 @@ implements org.apache.hadoop.util.Tool</pre>
<ul class="blockList">
<li class="blockList">
<h4>printUsage</h4>
-<pre>protected void <a href="../../../../../src-html/org/apache/hadoop/hbase/snapshot/ExportSnapshot.html#line.1112">printUsage</a>()</pre>
+<pre>protected void <a href="../../../../../src-html/org/apache/hadoop/hbase/snapshot/ExportSnapshot.html#line.1114">printUsage</a>()</pre>
<dl>
<dt><span class="overrideSpecifyLabel">Overrides:</span></dt>
<dd><code>printUsage</code> in class <code>org.apache.hadoop.hbase.util.AbstractHBaseTool</code></dd>
@@ -406,7 +406,7 @@ implements org.apache.hadoop.util.Tool</pre>
<ul class="blockList">
<li class="blockList">
<h4>addOptions</h4>
-<pre>protected void <a href="../../../../../src-html/org/apache/hadoop/hbase/snapshot/ExportSnapshot.html#line.1125">addOptions</a>()</pre>
+<pre>protected void <a href="../../../../../src-html/org/apache/hadoop/hbase/snapshot/ExportSnapshot.html#line.1127">addOptions</a>()</pre>
<div class="block"><span class="descfrmTypeLabel">Description copied from class: <code>org.apache.hadoop.hbase.util.AbstractHBaseTool</code></span></div>
<div class="block">Override this to add command-line options using <code>AbstractHBaseTool.addOptWithArg(java.lang.String, java.lang.String)</code>
and similar methods.</div>
@@ -422,7 +422,7 @@ implements org.apache.hadoop.util.Tool</pre>
<ul class="blockListLast">
<li class="blockList">
<h4>main</h4>
-<pre>public static void <a href="../../../../../src-html/org/apache/hadoop/hbase/snapshot/ExportSnapshot.html#line.1140">main</a>(<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a>[] args)</pre>
+<pre>public static void <a href="../../../../../src-html/org/apache/hadoop/hbase/snapshot/ExportSnapshot.html#line.1142">main</a>(<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a>[] args)</pre>
</li>
</ul>
</li>
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/705d69c4/apidocs/src-html/org/apache/hadoop/hbase/snapshot/ExportSnapshot.html
----------------------------------------------------------------------
diff --git a/apidocs/src-html/org/apache/hadoop/hbase/snapshot/ExportSnapshot.html b/apidocs/src-html/org/apache/hadoop/hbase/snapshot/ExportSnapshot.html
index 7d49582..01861a7 100644
--- a/apidocs/src-html/org/apache/hadoop/hbase/snapshot/ExportSnapshot.html
+++ b/apidocs/src-html/org/apache/hadoop/hbase/snapshot/ExportSnapshot.html
@@ -980,175 +980,177 @@
<span class="sourceLineNo">972</span> FileSystem outputFs = FileSystem.get(outputRoot.toUri(), destConf);<a name="line.972"></a>
<span class="sourceLineNo">973</span> LOG.debug("outputFs=" + outputFs.getUri().toString() + " outputRoot=" + outputRoot.toString());<a name="line.973"></a>
<span class="sourceLineNo">974</span><a name="line.974"></a>
-<span class="sourceLineNo">975</span> boolean skipTmp = conf.getBoolean(CONF_SKIP_TMP, false);<a name="line.975"></a>
-<span class="sourceLineNo">976</span><a name="line.976"></a>
-<span class="sourceLineNo">977</span> Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, inputRoot);<a name="line.977"></a>
-<span class="sourceLineNo">978</span> Path snapshotTmpDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(targetName, outputRoot);<a name="line.978"></a>
-<span class="sourceLineNo">979</span> Path outputSnapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(targetName, outputRoot);<a name="line.979"></a>
-<span class="sourceLineNo">980</span> Path initialOutputSnapshotDir = skipTmp ? outputSnapshotDir : snapshotTmpDir;<a name="line.980"></a>
-<span class="sourceLineNo">981</span><a name="line.981"></a>
-<span class="sourceLineNo">982</span> // Find the necessary directory which need to change owner and group<a name="line.982"></a>
-<span class="sourceLineNo">983</span> Path needSetOwnerDir = SnapshotDescriptionUtils.getSnapshotRootDir(outputRoot);<a name="line.983"></a>
-<span class="sourceLineNo">984</span> if (outputFs.exists(needSetOwnerDir)) {<a name="line.984"></a>
-<span class="sourceLineNo">985</span> if (skipTmp) {<a name="line.985"></a>
-<span class="sourceLineNo">986</span> needSetOwnerDir = outputSnapshotDir;<a name="line.986"></a>
-<span class="sourceLineNo">987</span> } else {<a name="line.987"></a>
-<span class="sourceLineNo">988</span> needSetOwnerDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(outputRoot);<a name="line.988"></a>
-<span class="sourceLineNo">989</span> if (outputFs.exists(needSetOwnerDir)) {<a name="line.989"></a>
-<span class="sourceLineNo">990</span> needSetOwnerDir = snapshotTmpDir;<a name="line.990"></a>
-<span class="sourceLineNo">991</span> }<a name="line.991"></a>
-<span class="sourceLineNo">992</span> }<a name="line.992"></a>
-<span class="sourceLineNo">993</span> }<a name="line.993"></a>
-<span class="sourceLineNo">994</span><a name="line.994"></a>
-<span class="sourceLineNo">995</span> // Check if the snapshot already exists<a name="line.995"></a>
-<span class="sourceLineNo">996</span> if (outputFs.exists(outputSnapshotDir)) {<a name="line.996"></a>
-<span class="sourceLineNo">997</span> if (overwrite) {<a name="line.997"></a>
-<span class="sourceLineNo">998</span> if (!outputFs.delete(outputSnapshotDir, true)) {<a name="line.998"></a>
-<span class="sourceLineNo">999</span> System.err.println("Unable to remove existing snapshot directory: " + outputSnapshotDir);<a name="line.999"></a>
-<span class="sourceLineNo">1000</span> return 1;<a name="line.1000"></a>
-<span class="sourceLineNo">1001</span> }<a name="line.1001"></a>
-<span class="sourceLineNo">1002</span> } else {<a name="line.1002"></a>
-<span class="sourceLineNo">1003</span> System.err.println("The snapshot '" + targetName +<a name="line.1003"></a>
-<span class="sourceLineNo">1004</span> "' already exists in the destination: " + outputSnapshotDir);<a name="line.1004"></a>
-<span class="sourceLineNo">1005</span> return 1;<a name="line.1005"></a>
-<span class="sourceLineNo">1006</span> }<a name="line.1006"></a>
-<span class="sourceLineNo">1007</span> }<a name="line.1007"></a>
-<span class="sourceLineNo">1008</span><a name="line.1008"></a>
-<span class="sourceLineNo">1009</span> if (!skipTmp) {<a name="line.1009"></a>
-<span class="sourceLineNo">1010</span> // Check if the snapshot already in-progress<a name="line.1010"></a>
-<span class="sourceLineNo">1011</span> if (outputFs.exists(snapshotTmpDir)) {<a name="line.1011"></a>
-<span class="sourceLineNo">1012</span> if (overwrite) {<a name="line.1012"></a>
-<span class="sourceLineNo">1013</span> if (!outputFs.delete(snapshotTmpDir, true)) {<a name="line.1013"></a>
-<span class="sourceLineNo">1014</span> System.err.println("Unable to remove existing snapshot tmp directory: "+snapshotTmpDir);<a name="line.1014"></a>
-<span class="sourceLineNo">1015</span> return 1;<a name="line.1015"></a>
-<span class="sourceLineNo">1016</span> }<a name="line.1016"></a>
-<span class="sourceLineNo">1017</span> } else {<a name="line.1017"></a>
-<span class="sourceLineNo">1018</span> System.err.println("A snapshot with the same name '"+ targetName +"' may be in-progress");<a name="line.1018"></a>
-<span class="sourceLineNo">1019</span> System.err.println("Please check "+snapshotTmpDir+". If the snapshot has completed, ");<a name="line.1019"></a>
-<span class="sourceLineNo">1020</span> System.err.println("consider removing "+snapshotTmpDir+" by using the -overwrite option");<a name="line.1020"></a>
-<span class="sourceLineNo">1021</span> return 1;<a name="line.1021"></a>
-<span class="sourceLineNo">1022</span> }<a name="line.1022"></a>
-<span class="sourceLineNo">1023</span> }<a name="line.1023"></a>
-<span class="sourceLineNo">1024</span> }<a name="line.1024"></a>
-<span class="sourceLineNo">1025</span><a name="line.1025"></a>
-<span class="sourceLineNo">1026</span> // Step 1 - Copy fs1:/.snapshot/<snapshot> to fs2:/.snapshot/.tmp/<snapshot><a name="line.1026"></a>
-<span class="sourceLineNo">1027</span> // The snapshot references must be copied before the hfiles otherwise the cleaner<a name="line.1027"></a>
-<span class="sourceLineNo">1028</span> // will remove them because they are unreferenced.<a name="line.1028"></a>
-<span class="sourceLineNo">1029</span> List<Path> travesedPaths = new ArrayList<>();<a name="line.1029"></a>
-<span class="sourceLineNo">1030</span> boolean copySucceeded = false;<a name="line.1030"></a>
-<span class="sourceLineNo">1031</span> try {<a name="line.1031"></a>
-<span class="sourceLineNo">1032</span> LOG.info("Copy Snapshot Manifest from " + snapshotDir + " to " + initialOutputSnapshotDir);<a name="line.1032"></a>
-<span class="sourceLineNo">1033</span> travesedPaths =<a name="line.1033"></a>
-<span class="sourceLineNo">1034</span> FSUtils.copyFilesParallel(inputFs, snapshotDir, outputFs, initialOutputSnapshotDir, conf,<a name="line.1034"></a>
-<span class="sourceLineNo">1035</span> conf.getInt(CONF_COPY_MANIFEST_THREADS, DEFAULT_COPY_MANIFEST_THREADS));<a name="line.1035"></a>
-<span class="sourceLineNo">1036</span> copySucceeded = true;<a name="line.1036"></a>
-<span class="sourceLineNo">1037</span> } catch (IOException e) {<a name="line.1037"></a>
-<span class="sourceLineNo">1038</span> throw new ExportSnapshotException("Failed to copy the snapshot directory: from=" +<a name="line.1038"></a>
-<span class="sourceLineNo">1039</span> snapshotDir + " to=" + initialOutputSnapshotDir, e);<a name="line.1039"></a>
-<span class="sourceLineNo">1040</span> } finally {<a name="line.1040"></a>
-<span class="sourceLineNo">1041</span> if (copySucceeded) {<a name="line.1041"></a>
-<span class="sourceLineNo">1042</span> if (filesUser != null || filesGroup != null) {<a name="line.1042"></a>
-<span class="sourceLineNo">1043</span> LOG.warn((filesUser == null ? "" : "Change the owner of " + needSetOwnerDir + " to "<a name="line.1043"></a>
-<span class="sourceLineNo">1044</span> + filesUser)<a name="line.1044"></a>
-<span class="sourceLineNo">1045</span> + (filesGroup == null ? "" : ", Change the group of " + needSetOwnerDir + " to "<a name="line.1045"></a>
-<span class="sourceLineNo">1046</span> + filesGroup));<a name="line.1046"></a>
-<span class="sourceLineNo">1047</span> setOwnerParallel(outputFs, filesUser, filesGroup, conf, travesedPaths);<a name="line.1047"></a>
-<span class="sourceLineNo">1048</span> }<a name="line.1048"></a>
-<span class="sourceLineNo">1049</span> if (filesMode > 0) {<a name="line.1049"></a>
-<span class="sourceLineNo">1050</span> LOG.warn("Change the permission of " + needSetOwnerDir + " to " + filesMode);<a name="line.1050"></a>
-<span class="sourceLineNo">1051</span> setPermissionParallel(outputFs, (short)filesMode, travesedPaths, conf);<a name="line.1051"></a>
-<span class="sourceLineNo">1052</span> }<a name="line.1052"></a>
-<span class="sourceLineNo">1053</span> }<a name="line.1053"></a>
-<span class="sourceLineNo">1054</span> }<a name="line.1054"></a>
-<span class="sourceLineNo">1055</span><a name="line.1055"></a>
-<span class="sourceLineNo">1056</span> // Write a new .snapshotinfo if the target name is different from the source name<a name="line.1056"></a>
-<span class="sourceLineNo">1057</span> if (!targetName.equals(snapshotName)) {<a name="line.1057"></a>
-<span class="sourceLineNo">1058</span> SnapshotDescription snapshotDesc =<a name="line.1058"></a>
-<span class="sourceLineNo">1059</span> SnapshotDescriptionUtils.readSnapshotInfo(inputFs, snapshotDir)<a name="line.1059"></a>
-<span class="sourceLineNo">1060</span> .toBuilder()<a name="line.1060"></a>
-<span class="sourceLineNo">1061</span> .setName(targetName)<a name="line.1061"></a>
-<span class="sourceLineNo">1062</span> .build();<a name="line.1062"></a>
-<span class="sourceLineNo">1063</span> SnapshotDescriptionUtils.writeSnapshotInfo(snapshotDesc, initialOutputSnapshotDir, outputFs);<a name="line.1063"></a>
-<span class="sourceLineNo">1064</span> if (filesUser != null || filesGroup != null) {<a name="line.1064"></a>
-<span class="sourceLineNo">1065</span> outputFs.setOwner(new Path(initialOutputSnapshotDir,<a name="line.1065"></a>
-<span class="sourceLineNo">1066</span> SnapshotDescriptionUtils.SNAPSHOTINFO_FILE), filesUser, filesGroup);<a name="line.1066"></a>
-<span class="sourceLineNo">1067</span> }<a name="line.1067"></a>
-<span class="sourceLineNo">1068</span> if (filesMode > 0) {<a name="line.1068"></a>
-<span class="sourceLineNo">1069</span> outputFs.setPermission(new Path(initialOutputSnapshotDir,<a name="line.1069"></a>
-<span class="sourceLineNo">1070</span> SnapshotDescriptionUtils.SNAPSHOTINFO_FILE), new FsPermission((short)filesMode));<a name="line.1070"></a>
-<span class="sourceLineNo">1071</span> }<a name="line.1071"></a>
-<span class="sourceLineNo">1072</span> }<a name="line.1072"></a>
-<span class="sourceLineNo">1073</span><a name="line.1073"></a>
-<span class="sourceLineNo">1074</span> // Step 2 - Start MR Job to copy files<a name="line.1074"></a>
-<span class="sourceLineNo">1075</span> // The snapshot references must be copied before the files otherwise the files gets removed<a name="line.1075"></a>
-<span class="sourceLineNo">1076</span> // by the HFileArchiver, since they have no references.<a name="line.1076"></a>
-<span class="sourceLineNo">1077</span> try {<a name="line.1077"></a>
-<span class="sourceLineNo">1078</span> runCopyJob(inputRoot, outputRoot, snapshotName, snapshotDir, verifyChecksum,<a name="line.1078"></a>
-<span class="sourceLineNo">1079</span> filesUser, filesGroup, filesMode, mappers, bandwidthMB);<a name="line.1079"></a>
-<span class="sourceLineNo">1080</span><a name="line.1080"></a>
-<span class="sourceLineNo">1081</span> LOG.info("Finalize the Snapshot Export");<a name="line.1081"></a>
-<span class="sourceLineNo">1082</span> if (!skipTmp) {<a name="line.1082"></a>
-<span class="sourceLineNo">1083</span> // Step 3 - Rename fs2:/.snapshot/.tmp/<snapshot> fs2:/.snapshot/<snapshot><a name="line.1083"></a>
-<span class="sourceLineNo">1084</span> if (!outputFs.rename(snapshotTmpDir, outputSnapshotDir)) {<a name="line.1084"></a>
-<span class="sourceLineNo">1085</span> throw new ExportSnapshotException("Unable to rename snapshot directory from=" +<a name="line.1085"></a>
-<span class="sourceLineNo">1086</span> snapshotTmpDir + " to=" + outputSnapshotDir);<a name="line.1086"></a>
-<span class="sourceLineNo">1087</span> }<a name="line.1087"></a>
-<span class="sourceLineNo">1088</span> }<a name="line.1088"></a>
-<span class="sourceLineNo">1089</span><a name="line.1089"></a>
-<span class="sourceLineNo">1090</span> // Step 4 - Verify snapshot integrity<a name="line.1090"></a>
-<span class="sourceLineNo">1091</span> if (verifyTarget) {<a name="line.1091"></a>
-<span class="sourceLineNo">1092</span> LOG.info("Verify snapshot integrity");<a name="line.1092"></a>
-<span class="sourceLineNo">1093</span> verifySnapshot(destConf, outputFs, outputRoot, outputSnapshotDir);<a name="line.1093"></a>
-<span class="sourceLineNo">1094</span> }<a name="line.1094"></a>
-<span class="sourceLineNo">1095</span><a name="line.1095"></a>
-<span class="sourceLineNo">1096</span> LOG.info("Export Completed: " + targetName);<a name="line.1096"></a>
-<span class="sourceLineNo">1097</span> return 0;<a name="line.1097"></a>
-<span class="sourceLineNo">1098</span> } catch (Exception e) {<a name="line.1098"></a>
-<span class="sourceLineNo">1099</span> LOG.error("Snapshot export failed", e);<a name="line.1099"></a>
-<span class="sourceLineNo">1100</span> if (!skipTmp) {<a name="line.1100"></a>
-<span class="sourceLineNo">1101</span> outputFs.delete(snapshotTmpDir, true);<a name="line.1101"></a>
-<span class="sourceLineNo">1102</span> }<a name="line.1102"></a>
-<span class="sourceLineNo">1103</span> outputFs.delete(outputSnapshotDir, true);<a name="line.1103"></a>
-<span class="sourceLineNo">1104</span> return 1;<a name="line.1104"></a>
-<span class="sourceLineNo">1105</span> } finally {<a name="line.1105"></a>
-<span class="sourceLineNo">1106</span> IOUtils.closeStream(inputFs);<a name="line.1106"></a>
-<span class="sourceLineNo">1107</span> IOUtils.closeStream(outputFs);<a name="line.1107"></a>
-<span class="sourceLineNo">1108</span> }<a name="line.1108"></a>
-<span class="sourceLineNo">1109</span> }<a name="line.1109"></a>
-<span class="sourceLineNo">1110</span><a name="line.1110"></a>
-<span class="sourceLineNo">1111</span> @Override<a name="line.1111"></a>
-<span class="sourceLineNo">1112</span> protected void printUsage() {<a name="line.1112"></a>
-<span class="sourceLineNo">1113</span> super.printUsage();<a name="line.1113"></a>
-<span class="sourceLineNo">1114</span> System.out.println("\n"<a name="line.1114"></a>
-<span class="sourceLineNo">1115</span> + "Examples:\n"<a name="line.1115"></a>
-<span class="sourceLineNo">1116</span> + " hbase snapshot export \\\n"<a name="line.1116"></a>
-<span class="sourceLineNo">1117</span> + " --snapshot MySnapshot --copy-to hdfs://srv2:8082/hbase \\\n"<a name="line.1117"></a>
-<span class="sourceLineNo">1118</span> + " --chuser MyUser --chgroup MyGroup --chmod 700 --mappers 16\n"<a name="line.1118"></a>
-<span class="sourceLineNo">1119</span> + "\n"<a name="line.1119"></a>
-<span class="sourceLineNo">1120</span> + " hbase snapshot export \\\n"<a name="line.1120"></a>
-<span class="sourceLineNo">1121</span> + " --snapshot MySnapshot --copy-from hdfs://srv2:8082/hbase \\\n"<a name="line.1121"></a>
-<span class="sourceLineNo">1122</span> + " --copy-to hdfs://srv1:50070/hbase");<a name="line.1122"></a>
-<span class="sourceLineNo">1123</span> }<a name="line.1123"></a>
-<span class="sourceLineNo">1124</span><a name="line.1124"></a>
-<span class="sourceLineNo">1125</span> @Override protected void addOptions() {<a name="line.1125"></a>
-<span class="sourceLineNo">1126</span> addRequiredOption(Options.SNAPSHOT);<a name="line.1126"></a>
-<span class="sourceLineNo">1127</span> addOption(Options.COPY_TO);<a name="line.1127"></a>
-<span class="sourceLineNo">1128</span> addOption(Options.COPY_FROM);<a name="line.1128"></a>
-<span class="sourceLineNo">1129</span> addOption(Options.TARGET_NAME);<a name="line.1129"></a>
-<span class="sourceLineNo">1130</span> addOption(Options.NO_CHECKSUM_VERIFY);<a name="line.1130"></a>
-<span class="sourceLineNo">1131</span> addOption(Options.NO_TARGET_VERIFY);<a name="line.1131"></a>
-<span class="sourceLineNo">1132</span> addOption(Options.OVERWRITE);<a name="line.1132"></a>
-<span class="sourceLineNo">1133</span> addOption(Options.CHUSER);<a name="line.1133"></a>
-<span class="sourceLineNo">1134</span> addOption(Options.CHGROUP);<a name="line.1134"></a>
-<span class="sourceLineNo">1135</span> addOption(Options.CHMOD);<a name="line.1135"></a>
-<span class="sourceLineNo">1136</span> addOption(Options.MAPPERS);<a name="line.1136"></a>
-<span class="sourceLineNo">1137</span> addOption(Options.BANDWIDTH);<a name="line.1137"></a>
-<span class="sourceLineNo">1138</span> }<a name="line.1138"></a>
-<span class="sourceLineNo">1139</span><a name="line.1139"></a>
-<span class="sourceLineNo">1140</span> public static void main(String[] args) {<a name="line.1140"></a>
-<span class="sourceLineNo">1141</span> new ExportSnapshot().doStaticMain(args);<a name="line.1141"></a>
-<span class="sourceLineNo">1142</span> }<a name="line.1142"></a>
-<span class="sourceLineNo">1143</span>}<a name="line.1143"></a>
+<span class="sourceLineNo">975</span> boolean skipTmp = conf.getBoolean(CONF_SKIP_TMP, false) ||<a name="line.975"></a>
+<span class="sourceLineNo">976</span> conf.get(SnapshotDescriptionUtils.SNAPSHOT_WORKING_DIR) != null;<a name="line.976"></a>
+<span class="sourceLineNo">977</span><a name="line.977"></a>
+<span class="sourceLineNo">978</span> Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, inputRoot);<a name="line.978"></a>
+<span class="sourceLineNo">979</span> Path snapshotTmpDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(targetName, outputRoot,<a name="line.979"></a>
+<span class="sourceLineNo">980</span> destConf);<a name="line.980"></a>
+<span class="sourceLineNo">981</span> Path outputSnapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(targetName, outputRoot);<a name="line.981"></a>
+<span class="sourceLineNo">982</span> Path initialOutputSnapshotDir = skipTmp ? outputSnapshotDir : snapshotTmpDir;<a name="line.982"></a>
+<span class="sourceLineNo">983</span><a name="line.983"></a>
+<span class="sourceLineNo">984</span> // Find the necessary directory which need to change owner and group<a name="line.984"></a>
+<span class="sourceLineNo">985</span> Path needSetOwnerDir = SnapshotDescriptionUtils.getSnapshotRootDir(outputRoot);<a name="line.985"></a>
+<span class="sourceLineNo">986</span> if (outputFs.exists(needSetOwnerDir)) {<a name="line.986"></a>
+<span class="sourceLineNo">987</span> if (skipTmp) {<a name="line.987"></a>
+<span class="sourceLineNo">988</span> needSetOwnerDir = outputSnapshotDir;<a name="line.988"></a>
+<span class="sourceLineNo">989</span> } else {<a name="line.989"></a>
+<span class="sourceLineNo">990</span> needSetOwnerDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(outputRoot, destConf);<a name="line.990"></a>
+<span class="sourceLineNo">991</span> if (outputFs.exists(needSetOwnerDir)) {<a name="line.991"></a>
+<span class="sourceLineNo">992</span> needSetOwnerDir = snapshotTmpDir;<a name="line.992"></a>
+<span class="sourceLineNo">993</span> }<a name="line.993"></a>
+<span class="sourceLineNo">994</span> }<a name="line.994"></a>
+<span class="sourceLineNo">995</span> }<a name="line.995"></a>
+<span class="sourceLineNo">996</span><a name="line.996"></a>
+<span class="sourceLineNo">997</span> // Check if the snapshot already exists<a name="line.997"></a>
+<span class="sourceLineNo">998</span> if (outputFs.exists(outputSnapshotDir)) {<a name="line.998"></a>
+<span class="sourceLineNo">999</span> if (overwrite) {<a name="line.999"></a>
+<span class="sourceLineNo">1000</span> if (!outputFs.delete(outputSnapshotDir, true)) {<a name="line.1000"></a>
+<span class="sourceLineNo">1001</span> System.err.println("Unable to remove existing snapshot directory: " + outputSnapshotDir);<a name="line.1001"></a>
+<span class="sourceLineNo">1002</span> return 1;<a name="line.1002"></a>
+<span class="sourceLineNo">1003</span> }<a name="line.1003"></a>
+<span class="sourceLineNo">1004</span> } else {<a name="line.1004"></a>
+<span class="sourceLineNo">1005</span> System.err.println("The snapshot '" + targetName +<a name="line.1005"></a>
+<span class="sourceLineNo">1006</span> "' already exists in the destination: " + outputSnapshotDir);<a name="line.1006"></a>
+<span class="sourceLineNo">1007</span> return 1;<a name="line.1007"></a>
+<span class="sourceLineNo">1008</span> }<a name="line.1008"></a>
+<span class="sourceLineNo">1009</span> }<a name="line.1009"></a>
+<span class="sourceLineNo">1010</span><a name="line.1010"></a>
+<span class="sourceLineNo">1011</span> if (!skipTmp) {<a name="line.1011"></a>
+<span class="sourceLineNo">1012</span> // Check if the snapshot already in-progress<a name="line.1012"></a>
+<span class="sourceLineNo">1013</span> if (outputFs.exists(snapshotTmpDir)) {<a name="line.1013"></a>
+<span class="sourceLineNo">1014</span> if (overwrite) {<a name="line.1014"></a>
+<span class="sourceLineNo">1015</span> if (!outputFs.delete(snapshotTmpDir, true)) {<a name="line.1015"></a>
+<span class="sourceLineNo">1016</span> System.err.println("Unable to remove existing snapshot tmp directory: "+snapshotTmpDir);<a name="line.1016"></a>
+<span class="sourceLineNo">1017</span> return 1;<a name="line.1017"></a>
+<span class="sourceLineNo">1018</span> }<a name="line.1018"></a>
+<span class="sourceLineNo">1019</span> } else {<a name="line.1019"></a>
+<span class="sourceLineNo">1020</span> System.err.println("A snapshot with the same name '"+ targetName +"' may be in-progress");<a name="line.1020"></a>
+<span class="sourceLineNo">1021</span> System.err.println("Please check "+snapshotTmpDir+". If the snapshot has completed, ");<a name="line.1021"></a>
+<span class="sourceLineNo">1022</span> System.err.println("consider removing "+snapshotTmpDir+" by using the -overwrite option");<a name="line.1022"></a>
+<span class="sourceLineNo">1023</span> return 1;<a name="line.1023"></a>
+<span class="sourceLineNo">1024</span> }<a name="line.1024"></a>
+<span class="sourceLineNo">1025</span> }<a name="line.1025"></a>
+<span class="sourceLineNo">1026</span> }<a name="line.1026"></a>
+<span class="sourceLineNo">1027</span><a name="line.1027"></a>
+<span class="sourceLineNo">1028</span> // Step 1 - Copy fs1:/.snapshot/<snapshot> to fs2:/.snapshot/.tmp/<snapshot><a name="line.1028"></a>
+<span class="sourceLineNo">1029</span> // The snapshot references must be copied before the hfiles otherwise the cleaner<a name="line.1029"></a>
+<span class="sourceLineNo">1030</span> // will remove them because they are unreferenced.<a name="line.1030"></a>
+<span class="sourceLineNo">1031</span> List<Path> travesedPaths = new ArrayList<>();<a name="line.1031"></a>
+<span class="sourceLineNo">1032</span> boolean copySucceeded = false;<a name="line.1032"></a>
+<span class="sourceLineNo">1033</span> try {<a name="line.1033"></a>
+<span class="sourceLineNo">1034</span> LOG.info("Copy Snapshot Manifest from " + snapshotDir + " to " + initialOutputSnapshotDir);<a name="line.1034"></a>
+<span class="sourceLineNo">1035</span> travesedPaths =<a name="line.1035"></a>
+<span class="sourceLineNo">1036</span> FSUtils.copyFilesParallel(inputFs, snapshotDir, outputFs, initialOutputSnapshotDir, conf,<a name="line.1036"></a>
+<span class="sourceLineNo">1037</span> conf.getInt(CONF_COPY_MANIFEST_THREADS, DEFAULT_COPY_MANIFEST_THREADS));<a name="line.1037"></a>
+<span class="sourceLineNo">1038</span> copySucceeded = true;<a name="line.1038"></a>
+<span class="sourceLineNo">1039</span> } catch (IOException e) {<a name="line.1039"></a>
+<span class="sourceLineNo">1040</span> throw new ExportSnapshotException("Failed to copy the snapshot directory: from=" +<a name="line.1040"></a>
+<span class="sourceLineNo">1041</span> snapshotDir + " to=" + initialOutputSnapshotDir, e);<a name="line.1041"></a>
+<span class="sourceLineNo">1042</span> } finally {<a name="line.1042"></a>
+<span class="sourceLineNo">1043</span> if (copySucceeded) {<a name="line.1043"></a>
+<span class="sourceLineNo">1044</span> if (filesUser != null || filesGroup != null) {<a name="line.1044"></a>
+<span class="sourceLineNo">1045</span> LOG.warn((filesUser == null ? "" : "Change the owner of " + needSetOwnerDir + " to "<a name="line.1045"></a>
+<span class="sourceLineNo">1046</span> + filesUser)<a name="line.1046"></a>
+<span class="sourceLineNo">1047</span> + (filesGroup == null ? "" : ", Change the group of " + needSetOwnerDir + " to "<a name="line.1047"></a>
+<span class="sourceLineNo">1048</span> + filesGroup));<a name="line.1048"></a>
+<span class="sourceLineNo">1049</span> setOwnerParallel(outputFs, filesUser, filesGroup, conf, travesedPaths);<a name="line.1049"></a>
+<span class="sourceLineNo">1050</span> }<a name="line.1050"></a>
+<span class="sourceLineNo">1051</span> if (filesMode > 0) {<a name="line.1051"></a>
+<span class="sourceLineNo">1052</span> LOG.warn("Change the permission of " + needSetOwnerDir + " to " + filesMode);<a name="line.1052"></a>
+<span class="sourceLineNo">1053</span> setPermissionParallel(outputFs, (short)filesMode, travesedPaths, conf);<a name="line.1053"></a>
+<span class="sourceLineNo">1054</span> }<a name="line.1054"></a>
+<span class="sourceLineNo">1055</span> }<a name="line.1055"></a>
+<span class="sourceLineNo">1056</span> }<a name="line.1056"></a>
+<span class="sourceLineNo">1057</span><a name="line.1057"></a>
+<span class="sourceLineNo">1058</span> // Write a new .snapshotinfo if the target name is different from the source name<a name="line.1058"></a>
+<span class="sourceLineNo">1059</span> if (!targetName.equals(snapshotName)) {<a name="line.1059"></a>
+<span class="sourceLineNo">1060</span> SnapshotDescription snapshotDesc =<a name="line.1060"></a>
+<span class="sourceLineNo">1061</span> SnapshotDescriptionUtils.readSnapshotInfo(inputFs, snapshotDir)<a name="line.1061"></a>
+<span class="sourceLineNo">1062</span> .toBuilder()<a name="line.1062"></a>
+<span class="sourceLineNo">1063</span> .setName(targetName)<a name="line.1063"></a>
+<span class="sourceLineNo">1064</span> .build();<a name="line.1064"></a>
+<span class="sourceLineNo">1065</span> SnapshotDescriptionUtils.writeSnapshotInfo(snapshotDesc, initialOutputSnapshotDir, outputFs);<a name="line.1065"></a>
+<span class="sourceLineNo">1066</span> if (filesUser != null || filesGroup != null) {<a name="line.1066"></a>
+<span class="sourceLineNo">1067</span> outputFs.setOwner(new Path(initialOutputSnapshotDir,<a name="line.1067"></a>
+<span class="sourceLineNo">1068</span> SnapshotDescriptionUtils.SNAPSHOTINFO_FILE), filesUser, filesGroup);<a name="line.1068"></a>
+<span class="sourceLineNo">1069</span> }<a name="line.1069"></a>
+<span class="sourceLineNo">1070</span> if (filesMode > 0) {<a name="line.1070"></a>
+<span class="sourceLineNo">1071</span> outputFs.setPermission(new Path(initialOutputSnapshotDir,<a name="line.1071"></a>
+<span class="sourceLineNo">1072</span> SnapshotDescriptionUtils.SNAPSHOTINFO_FILE), new FsPermission((short)filesMode));<a name="line.1072"></a>
+<span class="sourceLineNo">1073</span> }<a name="line.1073"></a>
+<span class="sourceLineNo">1074</span> }<a name="line.1074"></a>
+<span class="sourceLineNo">1075</span><a name="line.1075"></a>
+<span class="sourceLineNo">1076</span> // Step 2 - Start MR Job to copy files<a name="line.1076"></a>
+<span class="sourceLineNo">1077</span> // The snapshot references must be copied before the files otherwise the files gets removed<a name="line.1077"></a>
+<span class="sourceLineNo">1078</span> // by the HFileArchiver, since they have no references.<a name="line.1078"></a>
+<span class="sourceLineNo">1079</span> try {<a name="line.1079"></a>
+<span class="sourceLineNo">1080</span> runCopyJob(inputRoot, outputRoot, snapshotName, snapshotDir, verifyChecksum,<a name="line.1080"></a>
+<span class="sourceLineNo">1081</span> filesUser, filesGroup, filesMode, mappers, bandwidthMB);<a name="line.1081"></a>
+<span class="sourceLineNo">1082</span><a name="line.1082"></a>
+<span class="sourceLineNo">1083</span> LOG.info("Finalize the Snapshot Export");<a name="line.1083"></a>
+<span class="sourceLineNo">1084</span> if (!skipTmp) {<a name="line.1084"></a>
+<span class="sourceLineNo">1085</span> // Step 3 - Rename fs2:/.snapshot/.tmp/<snapshot> fs2:/.snapshot/<snapshot><a name="line.1085"></a>
+<span class="sourceLineNo">1086</span> if (!outputFs.rename(snapshotTmpDir, outputSnapshotDir)) {<a name="line.1086"></a>
+<span class="sourceLineNo">1087</span> throw new ExportSnapshotException("Unable to rename snapshot directory from=" +<a name="line.1087"></a>
+<span class="sourceLineNo">1088</span> snapshotTmpDir + " to=" + outputSnapshotDir);<a name="line.1088"></a>
+<span class="sourceLineNo">1089</span> }<a name="line.1089"></a>
+<span class="sourceLineNo">1090</span> }<a name="line.1090"></a>
+<span class="sourceLineNo">1091</span><a name="line.1091"></a>
+<span class="sourceLineNo">1092</span> // Step 4 - Verify snapshot integrity<a name="line.1092"></a>
+<span class="sourceLineNo">1093</span> if (verifyTarget) {<a name="line.1093"></a>
+<span class="sourceLineNo">1094</span> LOG.info("Verify snapshot integrity");<a name="line.1094"></a>
+<span class="sourceLineNo">1095</span> verifySnapshot(destConf, outputFs, outputRoot, outputSnapshotDir);<a name="line.1095"></a>
+<span class="sourceLineNo">1096</span> }<a name="line.1096"></a>
+<span class="sourceLineNo">1097</span><a name="line.1097"></a>
+<span class="sourceLineNo">1098</span> LOG.info("Export Completed: " + targetName);<a name="line.1098"></a>
+<span class="sourceLineNo">1099</span> return 0;<a name="line.1099"></a>
+<span class="sourceLineNo">1100</span> } catch (Exception e) {<a name="line.1100"></a>
+<span class="sourceLineNo">1101</span> LOG.error("Snapshot export failed", e);<a name="line.1101"></a>
+<span class="sourceLineNo">1102</span> if (!skipTmp) {<a name="line.1102"></a>
+<span class="sourceLineNo">1103</span> outputFs.delete(snapshotTmpDir, true);<a name="line.1103"></a>
+<span class="sourceLineNo">1104</span> }<a name="line.1104"></a>
+<span class="sourceLineNo">1105</span> outputFs.delete(outputSnapshotDir, true);<a name="line.1105"></a>
+<span class="sourceLineNo">1106</span> return 1;<a name="line.1106"></a>
+<span class="sourceLineNo">1107</span> } finally {<a name="line.1107"></a>
+<span class="sourceLineNo">1108</span> IOUtils.closeStream(inputFs);<a name="line.1108"></a>
+<span class="sourceLineNo">1109</span> IOUtils.closeStream(outputFs);<a name="line.1109"></a>
+<span class="sourceLineNo">1110</span> }<a name="line.1110"></a>
+<span class="sourceLineNo">1111</span> }<a name="line.1111"></a>
+<span class="sourceLineNo">1112</span><a name="line.1112"></a>
+<span class="sourceLineNo">1113</span> @Override<a name="line.1113"></a>
+<span class="sourceLineNo">1114</span> protected void printUsage() {<a name="line.1114"></a>
+<span class="sourceLineNo">1115</span> super.printUsage();<a name="line.1115"></a>
+<span class="sourceLineNo">1116</span> System.out.println("\n"<a name="line.1116"></a>
+<span class="sourceLineNo">1117</span> + "Examples:\n"<a name="line.1117"></a>
+<span class="sourceLineNo">1118</span> + " hbase snapshot export \\\n"<a name="line.1118"></a>
+<span class="sourceLineNo">1119</span> + " --snapshot MySnapshot --copy-to hdfs://srv2:8082/hbase \\\n"<a name="line.1119"></a>
+<span class="sourceLineNo">1120</span> + " --chuser MyUser --chgroup MyGroup --chmod 700 --mappers 16\n"<a name="line.1120"></a>
+<span class="sourceLineNo">1121</span> + "\n"<a name="line.1121"></a>
+<span class="sourceLineNo">1122</span> + " hbase snapshot export \\\n"<a name="line.1122"></a>
+<span class="sourceLineNo">1123</span> + " --snapshot MySnapshot --copy-from hdfs://srv2:8082/hbase \\\n"<a name="line.1123"></a>
+<span class="sourceLineNo">1124</span> + " --copy-to hdfs://srv1:50070/hbase");<a name="line.1124"></a>
+<span class="sourceLineNo">1125</span> }<a name="line.1125"></a>
+<span class="sourceLineNo">1126</span><a name="line.1126"></a>
+<span class="sourceLineNo">1127</span> @Override protected void addOptions() {<a name="line.1127"></a>
+<span class="sourceLineNo">1128</span> addRequiredOption(Options.SNAPSHOT);<a name="line.1128"></a>
+<span class="sourceLineNo">1129</span> addOption(Options.COPY_TO);<a name="line.1129"></a>
+<span class="sourceLineNo">1130</span> addOption(Options.COPY_FROM);<a name="line.1130"></a>
+<span class="sourceLineNo">1131</span> addOption(Options.TARGET_NAME);<a name="line.1131"></a>
+<span class="sourceLineNo">1132</span> addOption(Options.NO_CHECKSUM_VERIFY);<a name="line.1132"></a>
+<span class="sourceLineNo">1133</span> addOption(Options.NO_TARGET_VERIFY);<a name="line.1133"></a>
+<span class="sourceLineNo">1134</span> addOption(Options.OVERWRITE);<a name="line.1134"></a>
+<span class="sourceLineNo">1135</span> addOption(Options.CHUSER);<a name="line.1135"></a>
+<span class="sourceLineNo">1136</span> addOption(Options.CHGROUP);<a name="line.1136"></a>
+<span class="sourceLineNo">1137</span> addOption(Options.CHMOD);<a name="line.1137"></a>
+<span class="sourceLineNo">1138</span> addOption(Options.MAPPERS);<a name="line.1138"></a>
+<span class="sourceLineNo">1139</span> addOption(Options.BANDWIDTH);<a name="line.1139"></a>
+<span class="sourceLineNo">1140</span> }<a name="line.1140"></a>
+<span class="sourceLineNo">1141</span><a name="line.1141"></a>
+<span class="sourceLineNo">1142</span> public static void main(String[] args) {<a name="line.1142"></a>
+<span class="sourceLineNo">1143</span> new ExportSnapshot().doStaticMain(args);<a name="line.1143"></a>
+<span class="sourceLineNo">1144</span> }<a name="line.1144"></a>
+<span class="sourceLineNo">1145</span>}<a name="line.1145"></a>
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/705d69c4/book.html
----------------------------------------------------------------------
diff --git a/book.html b/book.html
index 780c40c..67fdd2a 100644
--- a/book.html
+++ b/book.html
@@ -4478,6 +4478,21 @@ Some configurations would only appear in source code; the only way to identify t
</dd>
</dl>
</div>
+<div id="hbase.snapshot.working.dir" class="dlist">
+<dl>
+<dt class="hdlist1"><code>hbase.snapshot.working.dir</code></dt>
+<dd>
+<div class="paragraph">
+<div class="title">Description</div>
+<p>Location where the snapshotting process will occur. The location of the completed snapshots will not change, but the temporary directory where the snapshot process occurs will be set to this location. This can be a separate filesystem than the root directory, for performance increase purposes. See HBASE-21098 for more information</p>
+</div>
+<div class="paragraph">
+<div class="title">Default</div>
+<p>none</p>
+</div>
+</dd>
+</dl>
+</div>
<div id="hbase.server.compactchecker.interval.multiplier" class="dlist">
<dl>
<dt class="hdlist1"><code>hbase.server.compactchecker.interval.multiplier</code></dt>
@@ -41269,7 +41284,7 @@ org/apache/hadoop/hbase/security/access/AccessControlClient.revoke:(Lorg/apache/
<div id="footer">
<div id="footer-text">
Version 3.0.0-SNAPSHOT<br>
-Last updated 2018-09-11 14:32:07 UTC
+Last updated 2018-09-12 14:33:08 UTC
</div>
</div>
</body>
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/705d69c4/bulk-loads.html
----------------------------------------------------------------------
diff --git a/bulk-loads.html b/bulk-loads.html
index aab7314..7bd97cb 100644
--- a/bulk-loads.html
+++ b/bulk-loads.html
@@ -7,7 +7,7 @@
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <meta name="Date-Revision-yyyymmdd" content="20180911" />
+ <meta name="Date-Revision-yyyymmdd" content="20180912" />
<meta http-equiv="Content-Language" content="en" />
<title>Apache HBase –
Bulk Loads in Apache HBase (TM)
@@ -306,7 +306,7 @@ under the License. -->
<a href="https://www.apache.org/">The Apache Software Foundation</a>.
All rights reserved.
- <li id="publishDate" class="pull-right">Last Published: 2018-09-11</li>
+ <li id="publishDate" class="pull-right">Last Published: 2018-09-12</li>
</p>
</div>