You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mina.apache.org by lg...@apache.org on 2020/09/26 06:55:12 UTC
[mina-sshd] 01/02: [SSHD-1086] Added SftpPathDirectoryScanner class
This is an automated email from the ASF dual-hosted git repository.
lgoldstein pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/mina-sshd.git
commit eb5c510505dd8a5d470252d21614f976b3572886
Author: Lyor Goldstein <lg...@apache.org>
AuthorDate: Fri Sep 25 19:11:09 2020 +0300
[SSHD-1086] Added SftpPathDirectoryScanner class
---
CHANGES.md | 1 +
docs/sftp.md | 38 +++++
.../org/apache/sshd/common/util/SelectorUtils.java | 75 ++++++---
.../sshd/common/util/io/DirectoryScanner.java | 34 ++++-
.../sshd/common/util/io/DirectoryScannerTest.java | 2 +-
.../sftp/client/fs/SftpPathDirectoryScanner.java | 94 ++++++++++++
.../client/fs/AbstractSftpFilesSystemSupport.java | 60 ++++++++
.../sftp/client/fs/SftpDirectoryScannersTest.java | 137 +++++++++++++++++
.../sshd/sftp/client/fs/SftpFileSystemTest.java | 167 +++++++--------------
9 files changed, 462 insertions(+), 146 deletions(-)
diff --git a/CHANGES.md b/CHANGES.md
index 23f04ef..1c35b7c 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -31,6 +31,7 @@ or `-key-file` command line option.
* [SSHD-1076](https://issues.apache.org/jira/browse/SSHD-1076) Break down `ClientUserAuthService#auth` method into several to allow for flexible override
* [SSHD-1077](https://issues.apache.org/jira/browse/SSHD-1077) Added command line option to request specific SFTP version in `SftpCommandMain`
* [SSHD-1079](https://issues.apache.org/jira/browse/SSHD-1079) Experimental async mode on the local port forwarder
+* [SSHD-1086](https://issues.apache.org/jira/browse/SSHD-1086) Added SFTP aware directory scanning helper classes
## Behavioral changes and enhancements
diff --git a/docs/sftp.md b/docs/sftp.md
index 7bbfdb6..e9e6778 100644
--- a/docs/sftp.md
+++ b/docs/sftp.md
@@ -416,6 +416,44 @@ UTF-8 is used. **Note:** the value can be a charset name or a `java.nio.charset.
```
+### SFTP aware directory scanners
+
+The framework provides special SFTP aware directory scanners that look for files (!) matching specific patterns. The
+scanners support *recursive* scanning of the directories based on the selected patterns.
+
+E.g. - let's assume the layout present below
+
+```
+ root
+ + --- a1.txt
+ + --- a2.csv
+ + sub1
+ +--- b1.txt
+ +--- b2.csv
+ + sub2
+ + --- c1.txt
+ + --- c2.csv
+```
+
+Then scan results from `root` are expected as follows for the given patterns
+
+* "**/*" - all the files - `[a1.txt, a1.csv, b1.txt, b1.csv, c1.txt, c2.csv]`
+* "**/*.txt" - only the ".txt" files - `[a1.txt, b1.txt, c1.txt]`
+* "*" - only the files at the root - `[a1.txt, a1.csv]`
+* "*.csv" - only `a1.csv` at the root
+
+**Note:** the scanner supports various patterns - including *regex* - see `DirectoryScanner` and `SelectorUtils`
+classes for supported patterns and matching - include case sensitive vs. insensitive match.
+
+```java
+ // Using an SftpPathDirectoryScanner
+ FileSystem fs = ... obtain an SFTP file system instance ...
+ Path rootDir = fs.getPath(...remote path...);
+ DirectoryScanner ds = new SftpPathDirectoryScanner(basedir, ...pattern...);
+ Collection<Path> matches = ds.scan();
+
+```
+
## Extensions
Both client and server support several of the SFTP extensions specified in various drafts:
diff --git a/sshd-common/src/main/java/org/apache/sshd/common/util/SelectorUtils.java b/sshd-common/src/main/java/org/apache/sshd/common/util/SelectorUtils.java
index 9cb0a71..3bf04fd 100644
--- a/sshd-common/src/main/java/org/apache/sshd/common/util/SelectorUtils.java
+++ b/sshd-common/src/main/java/org/apache/sshd/common/util/SelectorUtils.java
@@ -93,6 +93,11 @@ public final class SelectorUtils {
* "**".
*/
public static boolean matchPatternStart(String pattern, String str, boolean isCaseSensitive) {
+ return matchPath(pattern, str, File.separator, isCaseSensitive);
+ }
+
+ public static boolean matchPatternStart(
+ String pattern, String str, String separator, boolean isCaseSensitive) {
if ((pattern.length() > (REGEX_HANDLER_PREFIX.length() + PATTERN_HANDLER_SUFFIX.length() + 1))
&& pattern.startsWith(REGEX_HANDLER_PREFIX)
&& pattern.endsWith(PATTERN_HANDLER_SUFFIX)) {
@@ -106,14 +111,15 @@ public final class SelectorUtils {
pattern = pattern.substring(ANT_HANDLER_PREFIX.length(), pattern.length() - PATTERN_HANDLER_SUFFIX.length());
}
- String altStr = str.replace('\\', '/');
+ if (matchAntPathPatternStart(pattern, str, separator, isCaseSensitive)) {
+ return true;
+ }
- return matchAntPathPatternStart(pattern, str, File.separator, isCaseSensitive)
- || matchAntPathPatternStart(pattern, altStr, "/", isCaseSensitive);
+ return matchAntPathPatternStart(pattern, str.replace('\\', '/'), "/", isCaseSensitive);
}
}
- private static boolean matchAntPathPatternStart(
+ public static boolean matchAntPathPatternStart(
String pattern, String str, String separator, boolean isCaseSensitive) {
// When str starts with a File.separator, pattern has to start with a
// File.separator.
@@ -137,8 +143,7 @@ public final class SelectorUtils {
if (patDir.equals("**")) {
break;
}
- if (!match(patDir, strDirs.get(strIdxStart),
- isCaseSensitive)) {
+ if (!match(patDir, strDirs.get(strIdxStart), isCaseSensitive)) {
return false;
}
patIdxStart++;
@@ -175,7 +180,13 @@ public final class SelectorUtils {
* @return <code>true</code> if the pattern matches against the string, or <code>false</code>
* otherwise.
*/
- public static boolean matchPath(String pattern, String str, boolean isCaseSensitive) {
+ public static boolean matchPath(
+ String pattern, String str, boolean isCaseSensitive) {
+ return matchPath(pattern, str, File.separator, isCaseSensitive);
+ }
+
+ public static boolean matchPath(
+ String pattern, String str, String separator, boolean isCaseSensitive) {
if ((pattern.length() > (REGEX_HANDLER_PREFIX.length() + PATTERN_HANDLER_SUFFIX.length() + 1))
&& pattern.startsWith(REGEX_HANDLER_PREFIX)
&& pattern.endsWith(PATTERN_HANDLER_SUFFIX)) {
@@ -188,21 +199,27 @@ public final class SelectorUtils {
pattern = pattern.substring(ANT_HANDLER_PREFIX.length(), pattern.length() - PATTERN_HANDLER_SUFFIX.length());
}
- return matchAntPathPattern(pattern, str, isCaseSensitive);
+ return matchAntPathPattern(pattern, str, separator, isCaseSensitive);
}
}
- private static boolean matchAntPathPattern(String pattern, String str, boolean isCaseSensitive) {
- // When str starts with a File.separator, pattern has to start with a
- // File.separator.
- // When pattern starts with a File.separator, str has to start with a
- // File.separator.
- if (str.startsWith(File.separator) != pattern.startsWith(File.separator)) {
+ public static boolean matchAntPathPattern(
+ String pattern, String str, boolean isCaseSensitive) {
+ return matchAntPathPattern(pattern, str, File.separator, isCaseSensitive);
+ }
+
+ public static boolean matchAntPathPattern(
+ String pattern, String str, String separator, boolean isCaseSensitive) {
+ // When str starts with a file separator, pattern has to start with a
+ // file separator.
+ // When pattern starts with a file separator, str has to start with a
+ // file separator.
+ if (str.startsWith(separator) != pattern.startsWith(separator)) {
return false;
}
- List<String> patDirs = tokenizePath(pattern, File.separator);
- List<String> strDirs = tokenizePath(str, File.separator);
+ List<String> patDirs = tokenizePath(pattern, separator);
+ List<String> strDirs = tokenizePath(str, separator);
int patIdxStart = 0;
int patIdxEnd = patDirs.size() - 1;
@@ -215,19 +232,23 @@ public final class SelectorUtils {
if (patDir.equals("**")) {
break;
}
- if (!match(patDir, strDirs.get(strIdxStart),
- isCaseSensitive)) {
+
+ String subDir = strDirs.get(strIdxStart);
+ if (!match(patDir, subDir, isCaseSensitive)) {
patDirs = null;
strDirs = null;
return false;
}
+
patIdxStart++;
strIdxStart++;
}
+
if (strIdxStart > strIdxEnd) {
// String is exhausted
for (int i = patIdxStart; i <= patIdxEnd; i++) {
- if (!patDirs.get(i).equals("**")) {
+ String subPat = patDirs.get(i);
+ if (!subPat.equals("**")) {
patDirs = null;
strDirs = null;
return false;
@@ -249,19 +270,23 @@ public final class SelectorUtils {
if (patDir.equals("**")) {
break;
}
- if (!match(patDir, strDirs.get(strIdxEnd),
- isCaseSensitive)) {
+
+ String subDir = strDirs.get(strIdxEnd);
+ if (!match(patDir, subDir, isCaseSensitive)) {
patDirs = null;
strDirs = null;
return false;
}
+
patIdxEnd--;
strIdxEnd--;
}
+
if (strIdxStart > strIdxEnd) {
// String is exhausted
for (int i = patIdxStart; i <= patIdxEnd; i++) {
- if (!patDirs.get(i).equals("**")) {
+ String subPat = patDirs.get(i);
+ if (!subPat.equals("**")) {
patDirs = null;
strDirs = null;
return false;
@@ -273,7 +298,8 @@ public final class SelectorUtils {
while (patIdxStart != patIdxEnd && strIdxStart <= strIdxEnd) {
int patIdxTmp = -1;
for (int i = patIdxStart + 1; i <= patIdxEnd; i++) {
- if (patDirs.get(i).equals("**")) {
+ String subPat = patDirs.get(i);
+ if (subPat.equals("**")) {
patIdxTmp = i;
break;
}
@@ -312,7 +338,8 @@ public final class SelectorUtils {
}
for (int i = patIdxStart; i <= patIdxEnd; i++) {
- if (!patDirs.get(i).equals("**")) {
+ String subPat = patDirs.get(i);
+ if (!subPat.equals("**")) {
patDirs = null;
strDirs = null;
return false;
diff --git a/sshd-common/src/main/java/org/apache/sshd/common/util/io/DirectoryScanner.java b/sshd-common/src/main/java/org/apache/sshd/common/util/io/DirectoryScanner.java
index dfff44d..457dbe0 100644
--- a/sshd-common/src/main/java/org/apache/sshd/common/util/io/DirectoryScanner.java
+++ b/sshd-common/src/main/java/org/apache/sshd/common/util/io/DirectoryScanner.java
@@ -35,6 +35,7 @@ import java.util.stream.Collectors;
import org.apache.sshd.common.util.GenericUtils;
import org.apache.sshd.common.util.OsUtils;
import org.apache.sshd.common.util.SelectorUtils;
+import org.apache.sshd.common.util.ValidateUtils;
/**
* <p>
@@ -104,7 +105,7 @@ import org.apache.sshd.common.util.SelectorUtils;
* <p>
* Example of usage:
* </p>
- *
+ *
* <pre>
* String[] includes = { "**\\*.class" };
* String[] excludes = { "modules\\*\\**" };
@@ -134,17 +135,22 @@ public class DirectoryScanner {
/**
* The base directory to be scanned.
*/
- private Path basedir;
+ protected Path basedir;
/**
* The patterns for the files to be included.
*/
- private List<String> includePatterns;
+ protected List<String> includePatterns;
/**
* Whether or not the file system should be treated as a case sensitive one.
*/
- private boolean caseSensitive = OsUtils.isUNIX();
+ protected boolean caseSensitive = OsUtils.isUNIX();
+
+ /**
+ * The file separator to use to parse paths - default=local O/S separator
+ */
+ protected String separator = File.separator;
public DirectoryScanner() {
super();
@@ -214,6 +220,9 @@ public class DirectoryScanner {
.collect(Collectors.toCollection(() -> new ArrayList<>(includes.size()))));
}
+ /**
+ * @return Whether or not the file system should be treated as a case sensitive one.
+ */
public boolean isCaseSensitive() {
return caseSensitive;
}
@@ -223,6 +232,17 @@ public class DirectoryScanner {
}
/**
+ * @return The file separator to use to parse paths - default=local O/S separator
+ */
+ public String getSeparator() {
+ return separator;
+ }
+
+ public void setSeparator(String separator) {
+ this.separator = ValidateUtils.checkNotNullAndNotEmpty(separator, "No separator provided");
+ }
+
+ /**
* Scans the base directory for files which match at least one include pattern and don't match any exclude patterns.
* If there are selectors then the files must pass muster there, as well.
*
@@ -303,8 +323,9 @@ public class DirectoryScanner {
}
boolean cs = isCaseSensitive();
+ String sep = getSeparator();
for (String include : includes) {
- if (SelectorUtils.matchPath(include, name, cs)) {
+ if (SelectorUtils.matchPath(include, name, sep, cs)) {
return true;
}
}
@@ -326,8 +347,9 @@ public class DirectoryScanner {
}
boolean cs = isCaseSensitive();
+ String sep = getSeparator();
for (String include : includes) {
- if (SelectorUtils.matchPatternStart(include, name, cs)) {
+ if (SelectorUtils.matchPatternStart(include, name, sep, cs)) {
return true;
}
}
diff --git a/sshd-common/src/test/java/org/apache/sshd/common/util/io/DirectoryScannerTest.java b/sshd-common/src/test/java/org/apache/sshd/common/util/io/DirectoryScannerTest.java
index ded468c..1411609 100644
--- a/sshd-common/src/test/java/org/apache/sshd/common/util/io/DirectoryScannerTest.java
+++ b/sshd-common/src/test/java/org/apache/sshd/common/util/io/DirectoryScannerTest.java
@@ -79,7 +79,7 @@ public class DirectoryScannerTest extends JUnitTestSupport {
Files.createDirectories(rootDir);
List<Path> expected = new ArrayList<>();
- for (int level = 1; level <= Byte.SIZE; level++) {
+ for (int level = 1; level <= 8; level++) {
Path file = rootDir.resolve(Integer.toString(level) + (((level & 0x03) == 0) ? ".csv" : ".txt"));
Files.write(file, Collections.singletonList(file.toString()), StandardCharsets.UTF_8);
String name = Objects.toString(file.getFileName());
diff --git a/sshd-sftp/src/main/java/org/apache/sshd/sftp/client/fs/SftpPathDirectoryScanner.java b/sshd-sftp/src/main/java/org/apache/sshd/sftp/client/fs/SftpPathDirectoryScanner.java
new file mode 100644
index 0000000..6bc13de
--- /dev/null
+++ b/sshd-sftp/src/main/java/org/apache/sshd/sftp/client/fs/SftpPathDirectoryScanner.java
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sshd.sftp.client.fs;
+
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.stream.Collectors;
+
+import org.apache.sshd.common.util.GenericUtils;
+import org.apache.sshd.common.util.SelectorUtils;
+import org.apache.sshd.common.util.ValidateUtils;
+import org.apache.sshd.common.util.io.DirectoryScanner;
+
+/**
+ * An SFTP-aware {@link DirectoryScanner} that assumes all {@link Path}-s refer to SFTP remote ones and match patterns
+ * use "/" as their separator with case sensitive matching by default (though the latter can be modified).
+ *
+ * @author <a href="mailto:dev@mina.apache.org">Apache MINA SSHD Project</a>
+ */
+public class SftpPathDirectoryScanner extends DirectoryScanner {
+ public SftpPathDirectoryScanner() {
+ this(true);
+ }
+
+ public SftpPathDirectoryScanner(boolean caseSensitive) {
+ setSeparator("/");
+ setCaseSensitive(caseSensitive);
+ }
+
+ public SftpPathDirectoryScanner(Path dir) {
+ this(dir, Collections.emptyList());
+ }
+
+ public SftpPathDirectoryScanner(Path dir, String... includes) {
+ this(dir, GenericUtils.isEmpty(includes) ? Collections.emptyList() : Arrays.asList(includes));
+ }
+
+ public SftpPathDirectoryScanner(Path dir, Collection<String> includes) {
+ this();
+
+ setBasedir(dir);
+ setIncludes(includes);
+ }
+
+ @Override
+ public String getSeparator() {
+ return "/";
+ }
+
+ @Override
+ public void setSeparator(String separator) {
+ ValidateUtils.checkState("/".equals(separator), "Invalid separator: '%s'", separator);
+ super.setSeparator(separator);
+ }
+
+ @Override
+ public void setIncludes(Collection<String> includes) {
+ this.includePatterns = GenericUtils.isEmpty(includes)
+ ? Collections.emptyList()
+ : Collections.unmodifiableList(
+ includes.stream()
+ .map(v -> adjustPattern(v))
+ .collect(Collectors.toCollection(() -> new ArrayList<>(includes.size()))));
+ }
+
+ public static String adjustPattern(String pattern) {
+ pattern = pattern.trim();
+ if ((!pattern.startsWith(SelectorUtils.REGEX_HANDLER_PREFIX)) && pattern.endsWith("/")) {
+ return pattern + "**";
+ }
+
+ return pattern;
+ }
+}
diff --git a/sshd-sftp/src/test/java/org/apache/sshd/sftp/client/fs/AbstractSftpFilesSystemSupport.java b/sshd-sftp/src/test/java/org/apache/sshd/sftp/client/fs/AbstractSftpFilesSystemSupport.java
new file mode 100644
index 0000000..7ede88f
--- /dev/null
+++ b/sshd-sftp/src/test/java/org/apache/sshd/sftp/client/fs/AbstractSftpFilesSystemSupport.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sshd.sftp.client.fs;
+
+import java.io.IOException;
+import java.net.URI;
+import java.nio.file.FileSystem;
+import java.util.Collections;
+import java.util.Map;
+
+import org.apache.sshd.client.session.ClientSession;
+import org.apache.sshd.sftp.client.AbstractSftpClientTestSupport;
+import org.apache.sshd.sftp.client.SftpClientFactory;
+import org.apache.sshd.sftp.client.SftpVersionSelector;
+
+/**
+ * @author <a href="mailto:dev@mina.apache.org">Apache MINA SSHD Project</a>
+ */
+public abstract class AbstractSftpFilesSystemSupport extends AbstractSftpClientTestSupport {
+ protected AbstractSftpFilesSystemSupport() throws IOException {
+ super();
+ }
+
+ protected static FileSystem createSftpFileSystem(ClientSession session, SftpVersionSelector selector) throws IOException {
+ return SftpClientFactory.instance().createSftpFileSystem(session, selector);
+ }
+
+ protected URI createDefaultFileSystemURI() {
+ return createDefaultFileSystemURI(Collections.emptyMap());
+ }
+
+ protected URI createDefaultFileSystemURI(Map<String, ?> params) {
+ return createFileSystemURI(getCurrentTestName(), params);
+ }
+
+ protected static URI createFileSystemURI(String username, Map<String, ?> params) {
+ return createFileSystemURI(username, port, params);
+ }
+
+ protected static URI createFileSystemURI(String username, int port, Map<String, ?> params) {
+ return SftpFileSystemProvider.createFileSystemURI(TEST_LOCALHOST, port, username, username, params);
+ }
+}
diff --git a/sshd-sftp/src/test/java/org/apache/sshd/sftp/client/fs/SftpDirectoryScannersTest.java b/sshd-sftp/src/test/java/org/apache/sshd/sftp/client/fs/SftpDirectoryScannersTest.java
new file mode 100644
index 0000000..612afc8
--- /dev/null
+++ b/sshd-sftp/src/test/java/org/apache/sshd/sftp/client/fs/SftpDirectoryScannersTest.java
@@ -0,0 +1,137 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sshd.sftp.client.fs;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.FileSystem;
+import java.nio.file.FileSystems;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.AbstractMap.SimpleImmutableEntry;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.function.BiPredicate;
+
+import org.apache.sshd.common.util.io.DirectoryScanner;
+import org.apache.sshd.util.test.CommonTestSupportUtils;
+import org.junit.Before;
+import org.junit.FixMethodOrder;
+import org.junit.Test;
+import org.junit.runners.MethodSorters;
+
+/**
+ * @author <a href="mailto:dev@mina.apache.org">Apache MINA SSHD Project</a>
+ */
+@FixMethodOrder(MethodSorters.NAME_ASCENDING)
+public class SftpDirectoryScannersTest extends AbstractSftpFilesSystemSupport {
+ private static final BiPredicate<Path, Path> BY_FILE_NAME = (p1, p2) -> {
+ String n1 = Objects.toString(p1.getFileName());
+ String n2 = Objects.toString(p2.getFileName());
+ return Objects.equals(n1, n2);
+ };
+
+ public SftpDirectoryScannersTest() throws IOException {
+ super();
+ }
+
+ @Before
+ public void setUp() throws Exception {
+ setupServer();
+ }
+
+ @Test
+ public void testSftpPathDirectoryScannerDeepScanning() throws IOException {
+ testSftpPathDirectoryScanner(setupDeepScanning(), "**/*");
+ }
+
+ @Test
+ public void testSftpDirectoryScannerFileSuffixMatching() throws IOException {
+ testSftpPathDirectoryScanner(setupFileSuffixMatching(), "*.txt");
+ }
+
+ private void testSftpPathDirectoryScanner(
+ Map.Entry<String, List<Path>> setup, String pattern)
+ throws IOException {
+ List<Path> expected = setup.getValue();
+ List<Path> actual;
+ try (FileSystem fs = FileSystems.newFileSystem(createDefaultFileSystemURI(), Collections.emptyMap())) {
+ String remDirPath = setup.getKey();
+ Path basedir = fs.getPath(remDirPath);
+ DirectoryScanner ds = new SftpPathDirectoryScanner(basedir, pattern);
+ actual = ds.scan(() -> new ArrayList<>(expected.size()));
+ }
+ Collections.sort(actual);
+
+ assertListEquals(getCurrentTestName(), expected, actual, BY_FILE_NAME);
+ }
+
+ private Map.Entry<String, List<Path>> setupDeepScanning() throws IOException {
+ Path targetPath = detectTargetFolder();
+ Path rootDir = CommonTestSupportUtils.resolve(targetPath,
+ TEMP_SUBFOLDER_NAME, getClass().getSimpleName(), getCurrentTestName());
+ CommonTestSupportUtils.deleteRecursive(rootDir); // start fresh
+
+ List<Path> expected = new ArrayList<>();
+ Path curLevel = rootDir;
+ for (int level = 1; level <= 3; level++) {
+ Path dir = Files.createDirectories(curLevel.resolve(Integer.toString(level)));
+ expected.add(dir);
+ Path file = dir.resolve(Integer.toString(level) + ".txt");
+ Files.write(file, Collections.singletonList(file.toString()), StandardCharsets.UTF_8);
+
+ expected.add(file);
+ curLevel = dir;
+ }
+ Collections.sort(expected);
+
+ Path parentPath = targetPath.getParent();
+ String remFilePath = CommonTestSupportUtils.resolveRelativeRemotePath(parentPath, rootDir);
+
+ return new SimpleImmutableEntry<>(remFilePath, expected);
+ }
+
+ private Map.Entry<String, List<Path>> setupFileSuffixMatching() throws IOException {
+ Path targetPath = detectTargetFolder();
+ Path rootDir = CommonTestSupportUtils.resolve(targetPath,
+ TEMP_SUBFOLDER_NAME, getClass().getSimpleName(), getCurrentTestName());
+ CommonTestSupportUtils.deleteRecursive(rootDir); // start fresh
+ Files.createDirectories(rootDir);
+
+ List<Path> expected = new ArrayList<>();
+ for (int level = 1; level <= 8; level++) {
+ Path file = rootDir.resolve(Integer.toString(level) + (((level & 0x03) == 0) ? ".csv" : ".txt"));
+ Files.write(file, Collections.singletonList(file.toString()), StandardCharsets.UTF_8);
+ String name = Objects.toString(file.getFileName());
+ if (name.endsWith(".txt")) {
+ expected.add(file);
+ }
+ }
+ Collections.sort(expected);
+
+ Path parentPath = targetPath.getParent();
+ String remFilePath = CommonTestSupportUtils.resolveRelativeRemotePath(parentPath, rootDir);
+
+ return new SimpleImmutableEntry<>(remFilePath, expected);
+ }
+}
diff --git a/sshd-sftp/src/test/java/org/apache/sshd/sftp/client/fs/SftpFileSystemTest.java b/sshd-sftp/src/test/java/org/apache/sshd/sftp/client/fs/SftpFileSystemTest.java
index ac49dd4..f41ea52 100644
--- a/sshd-sftp/src/test/java/org/apache/sshd/sftp/client/fs/SftpFileSystemTest.java
+++ b/sshd-sftp/src/test/java/org/apache/sshd/sftp/client/fs/SftpFileSystemTest.java
@@ -60,69 +60,36 @@ import java.util.Map;
import java.util.TreeMap;
import java.util.concurrent.atomic.AtomicInteger;
-import org.apache.sshd.client.SshClient;
import org.apache.sshd.client.session.ClientSession;
-import org.apache.sshd.common.file.FileSystemFactory;
-import org.apache.sshd.common.file.virtualfs.VirtualFileSystemFactory;
import org.apache.sshd.common.session.Session;
import org.apache.sshd.common.util.GenericUtils;
import org.apache.sshd.common.util.MapEntryUtils.MapBuilder;
import org.apache.sshd.common.util.OsUtils;
import org.apache.sshd.common.util.io.IoUtils;
-import org.apache.sshd.server.SshServer;
import org.apache.sshd.sftp.SftpModuleProperties;
import org.apache.sshd.sftp.client.SftpClient;
-import org.apache.sshd.sftp.client.SftpClientFactory;
import org.apache.sshd.sftp.client.SftpVersionSelector;
import org.apache.sshd.sftp.common.SftpConstants;
import org.apache.sshd.sftp.server.SftpSubsystemEnvironment;
-import org.apache.sshd.sftp.server.SftpSubsystemFactory;
-import org.apache.sshd.util.test.BaseTestSupport;
import org.apache.sshd.util.test.CommonTestSupportUtils;
-import org.apache.sshd.util.test.CoreTestSupportUtils;
-import org.junit.AfterClass;
import org.junit.Before;
-import org.junit.BeforeClass;
import org.junit.FixMethodOrder;
import org.junit.Test;
import org.junit.runners.MethodSorters;
+/**
+ * @author <a href="mailto:dev@mina.apache.org">Apache MINA SSHD Project</a>
+ */
@FixMethodOrder(MethodSorters.NAME_ASCENDING)
@SuppressWarnings("checkstyle:MethodCount")
-public class SftpFileSystemTest extends BaseTestSupport {
- private static SshServer sshd;
- private static int port;
-
- private final FileSystemFactory fileSystemFactory;
-
+public class SftpFileSystemTest extends AbstractSftpFilesSystemSupport {
public SftpFileSystemTest() throws IOException {
- Path targetPath = detectTargetFolder();
- Path parentPath = targetPath.getParent();
- fileSystemFactory = new VirtualFileSystemFactory(parentPath);
- }
-
- @BeforeClass
- public static void setupServerInstance() throws Exception {
- sshd = CoreTestSupportUtils.setupTestServer(SftpFileSystemTest.class);
- sshd.setSubsystemFactories(Collections.singletonList(new SftpSubsystemFactory()));
- sshd.start();
- port = sshd.getPort();
- }
-
- @AfterClass
- public static void tearDownServerInstance() throws Exception {
- if (sshd != null) {
- try {
- sshd.stop(true);
- } finally {
- sshd = null;
- }
- }
+ super();
}
@Before
public void setUp() throws Exception {
- sshd.setFileSystemFactory(fileSystemFactory);
+ setupServer();
}
@Test
@@ -228,6 +195,8 @@ public class SftpFileSystemTest extends BaseTestSupport {
Path targetPath = detectTargetFolder();
Path lclSftp = CommonTestSupportUtils.resolve(targetPath,
SftpConstants.SFTP_SUBSYSTEM_NAME, getClass().getSimpleName());
+ Files.createDirectories(lclSftp);
+
Path lclFile = lclSftp.resolve(getCurrentTestName() + ".txt");
Files.deleteIfExists(lclFile);
byte[] expected
@@ -282,47 +251,41 @@ public class SftpFileSystemTest extends BaseTestSupport {
@Test
public void testMultipleFileStoresOnSameProvider() throws IOException {
- try (SshClient client = setupTestClient()) {
- client.start();
-
- SftpFileSystemProvider provider = new SftpFileSystemProvider(client);
- Collection<SftpFileSystem> fsList = new LinkedList<>();
- try {
- Collection<String> idSet = new HashSet<>();
- Map<String, Object> empty = Collections.emptyMap();
- for (int index = 0; index < 4; index++) {
- String credentials = getCurrentTestName() + "-user-" + index;
- SftpFileSystem expected = provider.newFileSystem(createFileSystemURI(credentials, empty), empty);
- fsList.add(expected);
-
- String id = expected.getId();
- assertTrue("Non unique file system id: " + id, idSet.add(id));
-
- SftpFileSystem actual = provider.getFileSystem(id);
- assertSame("Mismatched cached instances for " + id, expected, actual);
- outputDebugMessage("Created file system id: %s", id);
- }
+ SftpFileSystemProvider provider = new SftpFileSystemProvider(client);
+ Collection<SftpFileSystem> fsList = new LinkedList<>();
+ try {
+ Collection<String> idSet = new HashSet<>();
+ Map<String, Object> empty = Collections.emptyMap();
+ for (int index = 0; index < 4; index++) {
+ String credentials = getCurrentTestName() + "-user-" + index;
+ SftpFileSystem expected = provider.newFileSystem(createFileSystemURI(credentials, empty), empty);
+ fsList.add(expected);
+
+ String id = expected.getId();
+ assertTrue("Non unique file system id: " + id, idSet.add(id));
+
+ SftpFileSystem actual = provider.getFileSystem(id);
+ assertSame("Mismatched cached instances for " + id, expected, actual);
+ outputDebugMessage("Created file system id: %s", id);
+ }
- for (SftpFileSystem fs : fsList) {
- String id = fs.getId();
+ for (SftpFileSystem fs : fsList) {
+ String id = fs.getId();
+ fs.close();
+ assertNull("File system not removed from cache: " + id, provider.getFileSystem(id));
+ }
+ } finally {
+ IOException err = null;
+ for (FileSystem fs : fsList) {
+ try {
fs.close();
- assertNull("File system not removed from cache: " + id, provider.getFileSystem(id));
- }
- } finally {
- IOException err = null;
- for (FileSystem fs : fsList) {
- try {
- fs.close();
- } catch (IOException e) {
- err = GenericUtils.accumulateException(err, e);
- }
+ } catch (IOException e) {
+ err = GenericUtils.accumulateException(err, e);
}
+ }
- client.stop();
-
- if (err != null) {
- throw err;
- }
+ if (err != null) {
+ throw err;
}
}
}
@@ -341,25 +304,19 @@ public class SftpFileSystemTest extends BaseTestSupport {
return value;
};
- try (SshClient client = setupTestClient()) {
- client.start();
-
- try (ClientSession session = client.connect(getCurrentTestName(), TEST_LOCALHOST, port)
- .verify(CONNECT_TIMEOUT).getSession()) {
- session.addPasswordIdentity(getCurrentTestName());
- session.auth().verify(AUTH_TIMEOUT);
-
- try (FileSystem fs = createSftpFileSystem(session, selector)) {
- assertTrue("Not an SftpFileSystem", fs instanceof SftpFileSystem);
- Collection<String> views = fs.supportedFileAttributeViews();
- assertTrue("Universal views (" + SftpFileSystem.UNIVERSAL_SUPPORTED_VIEWS + ") not supported: " + views,
- views.containsAll(SftpFileSystem.UNIVERSAL_SUPPORTED_VIEWS));
- int expectedVersion = selected.get();
- assertEquals("Mismatched negotiated version", expectedVersion, ((SftpFileSystem) fs).getVersion());
- testFileSystem(fs, expectedVersion);
- }
- } finally {
- client.stop();
+ try (ClientSession session = client.connect(getCurrentTestName(), TEST_LOCALHOST, port)
+ .verify(CONNECT_TIMEOUT).getSession()) {
+ session.addPasswordIdentity(getCurrentTestName());
+ session.auth().verify(AUTH_TIMEOUT);
+
+ try (FileSystem fs = createSftpFileSystem(session, selector)) {
+ assertTrue("Not an SftpFileSystem", fs instanceof SftpFileSystem);
+ Collection<String> views = fs.supportedFileAttributeViews();
+ assertTrue("Universal views (" + SftpFileSystem.UNIVERSAL_SUPPORTED_VIEWS + ") not supported: " + views,
+ views.containsAll(SftpFileSystem.UNIVERSAL_SUPPORTED_VIEWS));
+ int expectedVersion = selected.get();
+ assertEquals("Mismatched negotiated version", expectedVersion, ((SftpFileSystem) fs).getVersion());
+ testFileSystem(fs, expectedVersion);
}
}
}
@@ -390,10 +347,6 @@ public class SftpFileSystemTest extends BaseTestSupport {
assertTrue("No configuration found", found);
}
- private FileSystem createSftpFileSystem(ClientSession session, SftpVersionSelector selector) throws IOException {
- return SftpClientFactory.instance().createSftpFileSystem(session, selector);
- }
-
private void testFileSystem(FileSystem fs, int version) throws Exception {
Iterable<Path> rootDirs = fs.getRootDirectories();
for (Path root : rootDirs) {
@@ -520,20 +473,4 @@ public class SftpFileSystemTest extends BaseTestSupport {
Files.delete(file1);
}
-
- private URI createDefaultFileSystemURI() {
- return createDefaultFileSystemURI(Collections.emptyMap());
- }
-
- private URI createDefaultFileSystemURI(Map<String, ?> params) {
- return createFileSystemURI(getCurrentTestName(), params);
- }
-
- private URI createFileSystemURI(String username, Map<String, ?> params) {
- return createFileSystemURI(username, port, params);
- }
-
- private static URI createFileSystemURI(String username, int port, Map<String, ?> params) {
- return SftpFileSystemProvider.createFileSystemURI(TEST_LOCALHOST, port, username, username, params);
- }
}