You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by er...@apache.org on 2017/03/27 19:18:15 UTC
lucene-solr:branch_6x: SLR-10108: bin/solr script recursive copy
broken
Repository: lucene-solr
Updated Branches:
refs/heads/branch_6x 6ecbe32dc -> c22c8bdeb
SLR-10108: bin/solr script recursive copy broken
(cherry picked from commit 0b3ca1b)
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/c22c8bde
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/c22c8bde
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/c22c8bde
Branch: refs/heads/branch_6x
Commit: c22c8bdebb9ab2a5cdb9951ccdec98a0f46f705e
Parents: 6ecbe32
Author: Erick Erickson <er...@apache.org>
Authored: Mon Mar 27 12:15:05 2017 -0700
Committer: Erick Erickson <er...@apache.org>
Committed: Mon Mar 27 12:18:06 2017 -0700
----------------------------------------------------------------------
solr/CHANGES.txt | 2 +
solr/bin/solr | 26 ++--
solr/bin/solr.cmd | 27 +++--
.../apache/solr/cloud/SolrCLIZkUtilsTest.java | 87 +++++++++++++-
.../solr/common/cloud/ZkMaintenanceUtils.java | 120 +++++++++++++++----
5 files changed, 214 insertions(+), 48 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c22c8bde/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index b84ea05..ef6ec37 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -60,6 +60,8 @@ Bug Fixes
* SOLR-10281: ADMIN_PATHS is duplicated in two places and inconsistent. This can cause automatic
retries to /admin/metrics handler by the CloudSolrClient. (shalin)
+* SOLR-10108: bin/solr script recursive copy broken (Erick Erickson)
+
Other Changes
----------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c22c8bde/solr/bin/solr
----------------------------------------------------------------------
diff --git a/solr/bin/solr b/solr/bin/solr
index 6e52ea2..bd3a0e2 100755
--- a/solr/bin/solr
+++ b/solr/bin/solr
@@ -492,17 +492,27 @@ function print_usage() {
echo " NOTE: <src> and <dest> may both be Zookeeper resources prefixed by 'zk:'"
echo " When <src> is a zk resource, <dest> may be '.'"
echo " If <dest> ends with '/', then <dest> will be a local folder or parent znode and the last"
- echo " element of the <src> path will be appended."
- echo ""
- echo " The 'file:' prefix is stripped, thus 'file:/' specifies an absolute local path and"
- echo " 'file:somewhere' specifies a relative local path. All paths on Zookeeper are absolute"
- echo " so the slash is required."
+ echo " element of the <src> path will be appended unless <src> also ends in a slash. "
+ echo " <dest> may be zk:, which may be useful when using the cp -r form to backup/restore "
+ echo " the entire zk state."
+ echo " You must enclose local paths that end in a wildcard in quotes or just"
+ echo " end the local path in a slash. That is,"
+ echo " 'bin/solr zk cp -r /some/dir/ zk:/ -z localhost:2181' is equivalent to"
+ echo " 'bin/solr zk cp -r \"/some/dir/*\" zk:/ -z localhost:2181'"
+ echo " but 'bin/solr zk cp -r /some/dir/* zk:/ -z localhost:2181' will throw an error"
+ echo ""
+ echo " here's an example of backup/restore for a ZK configuration:"
+ echo " to copy to local: 'bin/solr zk cp -r zk:/ /some/dir -z localhost:2181'"
+ echo " to restore to ZK: 'bin/solr zk cp -r /some/dir/ zk:/ -z localhost:2181'"
+ echo ""
+ echo " The 'file:' prefix is stripped, thus 'file:/wherever' specifies an absolute local path and"
+ echo " 'file:somewhere' specifies a relative local path. All paths on Zookeeper are absolute."
echo ""
echo " Zookeeper nodes CAN have data, so moving a single file to a parent znode"
echo " will overlay the data on the parent Znode so specifying the trailing slash"
- echo " is important."
+ echo " can be important."
echo ""
- echo " Wildcards are not supported"
+ echo " Wildcards are supported when copying from local, trailing only and must be quoted."
echo ""
echo " rm deletes files or folders on Zookeeper"
echo " -r�����Recursively delete if <path> is a directory. Command will fail if <path>"
@@ -1089,7 +1099,7 @@ if [[ "$SCRIPT_CMD" == "zk" ]]; then
if [ -z "$ZK_DST" ]; then
ZK_DST=$1
else
- print_short_zk_usage "Unrecognized or misplaced command $1"
+ print_short_zk_usage "Unrecognized or misplaced command $1. 'cp' with trailing asterisk requires quoting, see help text."
fi
fi
shift
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c22c8bde/solr/bin/solr.cmd
----------------------------------------------------------------------
diff --git a/solr/bin/solr.cmd b/solr/bin/solr.cmd
index 0ca0c3b..2547eb6 100644
--- a/solr/bin/solr.cmd
+++ b/solr/bin/solr.cmd
@@ -475,23 +475,32 @@ echo.
echo. ^<src^>, ^<dest^> : [file:][/]path/to/local/file or zk:/path/to/zk/node
echo NOTE: ^<src^> and ^<dest^> may both be Zookeeper resources prefixed by 'zk:'
echo When ^<src^> is a zk resource, ^<dest^> may be '.'
-echo If ^<dest^> ends with '/', then ^<dest^> will be a local folder or parent znode and the last
-echo element of the ^<src^> path will be appended.
+echo element of the ^<src^> path will be appended unless ^<src^> also ends in a slash.
+echo ^<dest^> may be zk:, which may be useful when using the cp -r form to backup/restore
+echo the entire zk state.
+echo You must enclose local paths that end in a wildcard in quotes or just
+echo end the local path in a slash. That is,
+echo 'bin/solr zk cp -r /some/dir/ zk:/ -z localhost:2181' is equivalent to
+echo 'bin/solr zk cp -r ^"/some/dir/*^" zk:/ -z localhost:2181'
+echo but 'bin/solr zk cp -r /some/dir/* zk:/ -z localhost:2181' will throw an error
+echo .
+echo here's an example of backup/restore for a ZK configuration:
+echo to copy to local: 'bin/solr zk cp -r zk:/ /some/dir -z localhost:2181'
+echo to restore to ZK: 'bin/solr zk cp -r /some/dir/ zk:/ -z localhost:2181'
echo.
-echo The 'file:' prefix is stripped, thus 'file:/' specifies an absolute local path and
-echo 'file:somewhere' specifies a relative local path. All paths on Zookeeper are absolute
-echo so the slash is required.
+echo The 'file:' prefix is stripped, thus 'file:/wherever' specifies an absolute local path and
+echo 'file:somewhere' specifies a relative local path. All paths on Zookeeper are absolute.
echo.
echo Zookeeper nodes CAN have data, so moving a single file to a parent znode
echo will overlay the data on the parent Znode so specifying the trailing slash
-echo is important.
+echo can be important.
echo.
-echo Wildcards are not supported
+echo Wildcards are supported when copying from local, trailing only and must be quoted.
echo.
echo rm deletes files or folders on Zookeeper
echo -r Recursively delete if ^<path^> is a directory. Command will fail if ^<path^>
echo has children and -r is not specified. Optional
-echo ^<path^> : [zk:]/path/to/zk/node. ^<path^> may not be the root ('/')"
+echo ^<path^> : [zk:]/path/to/zk/node. ^<path^> may not be the root ('/')
echo.
echo mv moves (renames) znodes on Zookeeper
echo ^<src^>, ^<dest^> : Zookeeper nodes, the 'zk:' prefix is optional.
@@ -508,7 +517,7 @@ echo.
echo Only the node names are listed, not data
echo.
echo mkroot makes a znode in Zookeeper with no data. Can be used to make a path of arbitrary
-echo depth but primarily intended to create a 'chroot'."
+echo depth but primarily intended to create a 'chroot'.
echo.
echo ^<path^>: The Zookeeper path to create. Leading slash is assumed if not present.
echo Intermediate nodes are created as needed if not present.
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c22c8bde/solr/core/src/test/org/apache/solr/cloud/SolrCLIZkUtilsTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/SolrCLIZkUtilsTest.java b/solr/core/src/test/org/apache/solr/cloud/SolrCLIZkUtilsTest.java
index 78be30b..35ba1d4 100644
--- a/solr/core/src/test/org/apache/solr/cloud/SolrCLIZkUtilsTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/SolrCLIZkUtilsTest.java
@@ -28,11 +28,14 @@ import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.SimpleFileVisitor;
import java.nio.file.attribute.BasicFileAttributes;
+import java.util.ArrayList;
+import java.util.List;
import org.apache.solr.common.cloud.SolrZkClient;
import org.apache.solr.common.cloud.ZkMaintenanceUtils;
import org.apache.solr.util.SolrCLI;
import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.data.Stat;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -131,7 +134,7 @@ public class SolrCLIZkUtilsTest extends SolrCloudTestCase {
Path configSet = TEST_PATH().resolve("configsets");
Path srcPathCheck = configSet.resolve("cloud-subdirs").resolve("conf");
-
+
copyConfigUp(configSet, "cloud-subdirs", "cp1");
// Now copy it somewhere else on ZK.
@@ -201,7 +204,6 @@ public class SolrCLIZkUtilsTest extends SolrCloudTestCase {
assertEquals("Copy should have succeeded.", 0, res);
verifyZkLocalPathsMatch(srcPathCheck, "/cp4");
-
// try with recurse not specified
args = new String[]{
"-src", "file:" + srcPathCheck.toAbsolutePath().toString(),
@@ -306,6 +308,70 @@ public class SolrCLIZkUtilsTest extends SolrCloudTestCase {
assertEquals("Copy from somewhere in ZK to ZK root should have succeeded.", 0, res);
assertTrue("Should have found znode /solrconfig.xml: ", zkClient.exists("/solrconfig.xml", true));
+ // Check that the form path/ works for copying files up. Should append the last bit of the source path to the dst
+ args = new String[]{
+ "-src", "file:" + srcPathCheck.toAbsolutePath().toString(),
+ "-dst", "zk:/cp7/",
+ "-recurse", "true",
+ "-zkHost", zkAddr,
+ };
+
+ res = cpTool.runTool(SolrCLI.processCommandLineArgs(SolrCLI.joinCommonAndToolOptions(cpTool.getOptions()), args));
+ assertEquals("Copy should have succeeded.", 0, res);
+ verifyZkLocalPathsMatch(srcPathCheck, "/cp7/" + srcPathCheck.getFileName().toString());
+
+ // Check for an intermediate ZNODE having content. You know cp7/stopwords is a parent node.
+ tmp = createTempDir("dirdata");
+ Path file = Paths.get(tmp.toAbsolutePath().toString(), "zknode.data");
+ List<String> lines = new ArrayList<>();
+ lines.add("{Some Arbitrary Data}");
+ Files.write(file, lines, Charset.forName("UTF-8"));
+ // First, just copy the data up the cp7 since it's a directory.
+ args = new String[]{
+ "-src", "file:" + file.toAbsolutePath().toString(),
+ "-dst", "zk:/cp7/conf/stopwords/",
+ "-recurse", "false",
+ "-zkHost", zkAddr,
+ };
+
+ res = cpTool.runTool(SolrCLI.processCommandLineArgs(SolrCLI.joinCommonAndToolOptions(cpTool.getOptions()), args));
+ assertEquals("Copy should have succeeded.", 0, res);
+
+ String content = new String(zkClient.getData("/cp7/conf/stopwords", null, null, true), StandardCharsets.UTF_8);
+ assertTrue("There should be content in the node! ", content.contains("{Some Arbitrary Data}"));
+
+
+ res = cpTool.runTool(SolrCLI.processCommandLineArgs(SolrCLI.joinCommonAndToolOptions(cpTool.getOptions()), args));
+ assertEquals("Copy should have succeeded.", 0, res);
+
+ tmp = createTempDir("cp8");
+ args = new String[]{
+ "-src", "zk:/cp7",
+ "-dst", "file:" + tmp.toAbsolutePath().toString(),
+ "-recurse", "true",
+ "-zkHost", zkAddr,
+ };
+ res = cpTool.runTool(SolrCLI.processCommandLineArgs(SolrCLI.joinCommonAndToolOptions(cpTool.getOptions()), args));
+ assertEquals("Copy should have succeeded.", 0, res);
+
+ // Next, copy cp7 down and verify that zknode.data exists for cp7
+ Path zData = Paths.get(tmp.toAbsolutePath().toString(), "conf/stopwords/zknode.data");
+ assertTrue("znode.data should have been copied down", zData.toFile().exists());
+
+ // Finally, copy up to cp8 and verify that the data is up there.
+ args = new String[]{
+ "-src", "file:" + tmp.toAbsolutePath().toString(),
+ "-dst", "zk:/cp9",
+ "-recurse", "true",
+ "-zkHost", zkAddr,
+ };
+
+ res = cpTool.runTool(SolrCLI.processCommandLineArgs(SolrCLI.joinCommonAndToolOptions(cpTool.getOptions()), args));
+ assertEquals("Copy should have succeeded.", 0, res);
+
+ content = new String(zkClient.getData("/cp9/conf/stopwords", null, null, true), StandardCharsets.UTF_8);
+ assertTrue("There should be content in the node! ", content.contains("{Some Arbitrary Data}"));
+
}
@Test
@@ -577,13 +643,22 @@ public class SolrCLIZkUtilsTest extends SolrCloudTestCase {
verifyAllZNodesAreFiles(fileRoot, zkRoot);
}
+ private static boolean isEphemeral(String zkPath) throws KeeperException, InterruptedException {
+ Stat znodeStat = zkClient.exists(zkPath, null, true);
+ return znodeStat.getEphemeralOwner() != 0;
+ }
+
void verifyAllZNodesAreFiles(Path fileRoot, String zkRoot) throws KeeperException, InterruptedException {
- for (String node : zkClient.getChildren(zkRoot, null, true)) {
- Path thisPath = Paths.get(fileRoot.toAbsolutePath().toString(), node);
- assertTrue("Znode " + node + " should have been found on disk at " + fileRoot.toAbsolutePath().toString(),
+ for (String child : zkClient.getChildren(zkRoot, null, true)) {
+ // Skip ephemeral nodes
+ if (zkRoot.endsWith("/") == false) zkRoot += "/";
+ if (isEphemeral(zkRoot + child)) continue;
+
+ Path thisPath = Paths.get(fileRoot.toAbsolutePath().toString(), child);
+ assertTrue("Znode " + child + " should have been found on disk at " + fileRoot.toAbsolutePath().toString(),
Files.exists(thisPath));
- verifyAllZNodesAreFiles(thisPath, zkRoot + "/" + node);
+ verifyAllZNodesAreFiles(thisPath, zkRoot + child);
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c22c8bde/solr/solrj/src/java/org/apache/solr/common/cloud/ZkMaintenanceUtils.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/common/cloud/ZkMaintenanceUtils.java b/solr/solrj/src/java/org/apache/solr/common/cloud/ZkMaintenanceUtils.java
index b7aa3d2..f569ae3 100644
--- a/solr/solrj/src/java/org/apache/solr/common/cloud/ZkMaintenanceUtils.java
+++ b/solr/solrj/src/java/org/apache/solr/common/cloud/ZkMaintenanceUtils.java
@@ -17,6 +17,7 @@
package org.apache.solr.common.cloud;
+import java.io.File;
import java.io.IOException;
import java.lang.invoke.MethodHandles;
import java.nio.file.FileVisitResult;
@@ -31,6 +32,7 @@ import java.util.regex.Pattern;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.data.Stat;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -40,6 +42,7 @@ import org.slf4j.LoggerFactory;
*/
public class ZkMaintenanceUtils {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+ private static final String ZKNODE_DATA_FILE = "zknode.data";
private ZkMaintenanceUtils() {} // don't let it be instantiated, all methods are static.
/**
@@ -119,6 +122,9 @@ public class ZkMaintenanceUtils {
if (srcIsZk == false && dstIsZk == false) {
throw new SolrServerException("At least one of the source and dest parameters must be prefixed with 'zk:' ");
}
+ if (dstIsZk && dst.length() == 0) {
+ dst = "/"; // for consistency, one can copy from zk: and send to zk:/
+ }
dst = normalizeDest(src, dst);
if (srcIsZk && dstIsZk) {
@@ -148,18 +154,26 @@ public class ZkMaintenanceUtils {
Files.write(filename, data);
}
+
private static String normalizeDest(String srcName, String dstName) {
- // Pull the last element of the src path and add it to the dst.
- if (dstName.endsWith("/")) {
+ // Special handling for "."
+ if (dstName.equals(".")) {
+ return Paths.get(".").normalize().toAbsolutePath().toString();
+ }
+ // Pull the last element of the src path and add it to the dst if the src does NOT end in a slash
+
+ // If the source ends in a slash, do not append the last segment to the dest
+
+ if (dstName.endsWith("/")) { // Dest is a directory.
int pos = srcName.lastIndexOf("/");
if (pos < 0) {
dstName += srcName;
} else {
dstName += srcName.substring(pos + 1);
}
- } else if (dstName.equals(".")) {
- dstName = Paths.get(".").normalize().toAbsolutePath().toString();
}
+
+ log.info("copying from '{}' to '{}'", srcName, dstName);
return dstName;
}
@@ -226,10 +240,17 @@ public class ZkMaintenanceUtils {
}
});
}
+
+ public static void uploadToZK(SolrZkClient zkClient, final Path fromPath, final String zkPath,
+ final Pattern filenameExclusions) throws IOException {
- public static void uploadToZK(SolrZkClient zkClient, final Path rootPath, final String zkPath,
- final Pattern filenameExclusions) throws IOException {
+ String path = fromPath.toString();
+ if (path.endsWith("*")) {
+ path = path.substring(0, path.length() - 1);
+ }
+ final Path rootPath = Paths.get(path);
+
if (!Files.exists(rootPath))
throw new IOException("Path " + rootPath + " does not exist");
@@ -243,7 +264,12 @@ public class ZkMaintenanceUtils {
}
String zkNode = createZkNodeName(zkPath, rootPath, file);
try {
- zkClient.makePath(zkNode, file.toFile(), false, true);
+ // if the path exists (and presumably we're uploading data to it) just set its data
+ if (file.toFile().getName().equals(ZKNODE_DATA_FILE) && zkClient.exists(zkNode, true)) {
+ zkClient.setData(zkNode, file.toFile(), true);
+ } else {
+ zkClient.makePath(zkNode, file.toFile(), false, true);
+ }
} catch (KeeperException | InterruptedException e) {
throw new IOException("Error uploading file " + file.toString() + " to zookeeper path " + zkNode,
SolrZkClient.checkInterrupted(e));
@@ -253,28 +279,58 @@ public class ZkMaintenanceUtils {
@Override
public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException {
- return (dir.getFileName().toString().startsWith(".")) ? FileVisitResult.SKIP_SUBTREE : FileVisitResult.CONTINUE;
+ if (dir.getFileName().toString().startsWith(".")) return FileVisitResult.SKIP_SUBTREE;
+
+ return FileVisitResult.CONTINUE;
}
});
}
- public static void downloadFromZK(SolrZkClient zkClient, String zkPath, Path dir) throws IOException {
+ private static boolean isEphemeral(SolrZkClient zkClient, String zkPath) throws KeeperException, InterruptedException {
+ Stat znodeStat = zkClient.exists(zkPath, null, true);
+ return znodeStat.getEphemeralOwner() != 0;
+ }
+
+ private static int copyDataDown(SolrZkClient zkClient, String zkPath, File file) throws IOException, KeeperException, InterruptedException {
+ byte[] data = zkClient.getData(zkPath, null, null, true);
+ if (data != null && data.length > 1) { // There are apparently basically empty ZNodes.
+ log.info("Writing file {}", file.toString());
+ Files.write(file.toPath(), data);
+ return data.length;
+ }
+ return 0;
+ }
+
+ public static void downloadFromZK(SolrZkClient zkClient, String zkPath, Path file) throws IOException {
try {
- List<String> files = zkClient.getChildren(zkPath, null, true);
- Files.createDirectories(dir);
- for (String file : files) {
- List<String> children = zkClient.getChildren(zkPath + "/" + file, null, true);
- if (children.size() == 0) {
- byte[] data = zkClient.getData(zkPath + "/" + file, null, null, true);
- Path filename = dir.resolve(file);
- log.info("Writing file {}", filename);
- Files.write(filename, data);
- } else {
- downloadFromZK(zkClient, zkPath + "/" + file, dir.resolve(file));
+ List<String> children = zkClient.getChildren(zkPath, null, true);
+ // If it has no children, it's a leaf node, write the assoicated data from the ZNode.
+ // Otherwise, continue recursing, but write the associated data to a special file if any
+ if (children.size() == 0) {
+ // If we didn't copy data down, then we also didn't create the file. But we still need a marker on the local
+ // disk so create a dir.
+ if (copyDataDown(zkClient, zkPath, file.toFile()) == 0) {
+ Files.createDirectories(file);
+ }
+ } else {
+ Files.createDirectories(file); // Make parent dir.
+ // ZK nodes, whether leaf or not can have data. If it's a non-leaf node and
+ // has associated data write it into the special file.
+ copyDataDown(zkClient, zkPath, new File(file.toFile(), ZKNODE_DATA_FILE));
+
+ for (String child : children) {
+ String zkChild = zkPath;
+ if (zkChild.endsWith("/") == false) zkChild += "/";
+ zkChild += child;
+ if (isEphemeral(zkClient, zkChild)) { // Don't copy ephemeral nodes
+ continue;
+ }
+ // Go deeper into the tree now
+ downloadFromZK(zkClient, zkChild, file.resolve(child));
}
}
} catch (KeeperException | InterruptedException e) {
- throw new IOException("Error downloading files from zookeeper path " + zkPath + " to " + dir.toString(),
+ throw new IOException("Error downloading files from zookeeper path " + zkPath + " to " + file.toString(),
SolrZkClient.checkInterrupted(e));
}
}
@@ -336,10 +392,24 @@ public class ZkMaintenanceUtils {
if ("\\".equals(separator))
relativePath = relativePath.replaceAll("\\\\", "/");
// It's possible that the relative path and file are the same, in which case
- // adding the bare slash is A Bad Idea
- if (relativePath.length() == 0) return zkRoot;
-
- return zkRoot + "/" + relativePath;
+ // adding the bare slash is A Bad Idea unless it's a non-leaf data node
+ boolean isNonLeafData = file.toFile().getName().equals(ZKNODE_DATA_FILE);
+ if (relativePath.length() == 0 && isNonLeafData == false) return zkRoot;
+
+ // Important to have this check if the source is file:whatever/ and the destination is just zk:/
+ if (zkRoot.endsWith("/") == false) zkRoot += "/";
+
+ String ret = zkRoot + relativePath;
+
+ // Special handling for data associated with non-leaf node.
+ if (isNonLeafData) {
+ // special handling since what we need to do is add the data to the parent.
+ ret = ret.substring(0, ret.indexOf(ZKNODE_DATA_FILE));
+ if (ret.endsWith("/")) {
+ ret = ret.substring(0, ret.length() - 1);
+ }
+ }
+ return ret;
}
}