You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mina.apache.org by lg...@apache.org on 2020/08/18 05:49:08 UTC
[mina-sshd] 05/06: [SSHD-1005] Added ScpTransferHelper support
This is an automated email from the ASF dual-hosted git repository.
lgoldstein pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/mina-sshd.git
commit 2be166000b540313337e3c8f9c3d61af74611284
Author: Lyor Goldstein <lg...@apache.org>
AuthorDate: Fri Aug 14 18:57:40 2020 +0300
[SSHD-1005] Added ScpTransferHelper support
---
CHANGES.md | 1 +
docs/scp.md | 15 ++
.../scp/client/ScpRemote2RemoteTransferHelper.java | 254 +++++++++++++++++++++
.../client/ScpRemote2RemoteTransferListener.java | 68 ++++++
.../org/apache/sshd/scp/server/ScpCommand.java | 2 +-
.../sshd/scp/client/AbstractScpTestSupport.java | 158 +++++++++++++
.../client/ScpRemote2RemoteTransferHelperTest.java | 124 ++++++++++
.../java/org/apache/sshd/scp/client/ScpTest.java | 190 ++-------------
8 files changed, 644 insertions(+), 168 deletions(-)
diff --git a/CHANGES.md b/CHANGES.md
index bd4bae8..33ba98d 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -31,6 +31,7 @@ or `-key-file` command line option.
* [SSHD-1004](https://issues.apache.org/jira/browse/SSHD-1004) Deprecate DES, RC4 and Blowfish ciphers from default setup.
* [SSHD-1004](https://issues.apache.org/jira/browse/SSHD-1004) Deprecate SHA-1 based key exchanges and signatures from default setup.
* [SSHD-1004](https://issues.apache.org/jira/browse/SSHD-1004) Deprecate MD5-based and truncated HMAC algorithms from default setup.
+* [SSHD-1005](https://issues.apache.org/jira/browse/SSHD-1005) Added support for SCP remote-to-remote file transfer
* [SSHD-1020](https://issues.apache.org/jira/browse/SSHD-1020) SSH connections getting closed abruptly with timeout exceptions.
* [SSHD-1026](https://issues.apache.org/jira/browse/SSHD-1026) Improve build reproductibility.
* [SSHD-1028](https://issues.apache.org/jira/browse/SSHD-1028) Fix SSH_MSG_DISCONNECT: Too many concurrent connections.
diff --git a/docs/scp.md b/docs/scp.md
index a91a09d..499241c 100644
--- a/docs/scp.md
+++ b/docs/scp.md
@@ -172,6 +172,21 @@ is likely to require. For this purpose, the `ScpCommandFactory` also implements
**Note:** a similar result can be achieved if activating SSHD from the command line by specifying `-o ShellFactory=scp`
+## Remote-to-remote transfer
+
+The code provides an `ScpTransferHelper` class that enables copying files between 2 remote accounts without going through
+the local file system.
+
+```java
+ ClientSession src = ...;
+ ClientSession dst = ...;
+ // Can also provide a listener in the constructor in order to be informed of the actual transfer progress
+ ScpRemote2RemoteTransferHelper helper = new ScpRemote2RemoteTransferHelper(src, dst);
+ // can be repeated for as many files as necessary using the same helper
+ helper.transferFile("remote/src/path/file1", "remote/dst/path/file2");
+
+```
+
## References
* [How the SCP protocol works](https://chuacw.ath.cx/development/b/chuacw/archive/2019/02/04/how-the-scp-protocol-works.aspx)
diff --git a/sshd-scp/src/main/java/org/apache/sshd/scp/client/ScpRemote2RemoteTransferHelper.java b/sshd-scp/src/main/java/org/apache/sshd/scp/client/ScpRemote2RemoteTransferHelper.java
new file mode 100644
index 0000000..1b4852a
--- /dev/null
+++ b/sshd-scp/src/main/java/org/apache/sshd/scp/client/ScpRemote2RemoteTransferHelper.java
@@ -0,0 +1,254 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sshd.scp.client;
+
+import java.io.EOFException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.io.StreamCorruptedException;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.EnumSet;
+import java.util.Objects;
+
+import org.apache.sshd.client.channel.ChannelExec;
+import org.apache.sshd.client.session.ClientSession;
+import org.apache.sshd.common.util.io.IoUtils;
+import org.apache.sshd.common.util.io.LimitInputStream;
+import org.apache.sshd.common.util.logging.AbstractLoggingBean;
+import org.apache.sshd.scp.client.ScpClient.Option;
+import org.apache.sshd.scp.common.ScpTimestamp;
+import org.apache.sshd.scp.common.helpers.AbstractScpCommandDetails;
+import org.apache.sshd.scp.common.helpers.ScpIoUtils;
+import org.apache.sshd.scp.common.helpers.ScpReceiveFileCommandDetails;
+
+/**
+ * Helps transfer files between 2 servers rather than between server and local file system by using 2
+ * {@link ClientSession}-s and simply copying from one server to the other
+ *
+ * @author <a href="mailto:dev@mina.apache.org">Apache MINA SSHD Project</a>
+ */
+public class ScpRemote2RemoteTransferHelper extends AbstractLoggingBean {
+ protected final ScpRemote2RemoteTransferListener listener;
+
+ private final ClientSession sourceSession;
+ private final ClientSession destSession;
+
+ public ScpRemote2RemoteTransferHelper(ClientSession sourceSession, ClientSession destSession) {
+ this(sourceSession, destSession, null);
+ }
+
+ /**
+ * @param sourceSession The source {@link ClientSession}
+ * @param destSession The destination {@link ClientSession}
+ * @param listener An optional {@link ScpRemote2RemoteTransferListener}
+ */
+ public ScpRemote2RemoteTransferHelper(ClientSession sourceSession, ClientSession destSession,
+ ScpRemote2RemoteTransferListener listener) {
+ this.sourceSession = Objects.requireNonNull(sourceSession, "No source session provided");
+ this.destSession = Objects.requireNonNull(destSession, "No destination session provided");
+ this.listener = listener;
+ }
+
+ public ClientSession getSourceSession() {
+ return sourceSession;
+ }
+
+ public ClientSession getDestinationSession() {
+ return destSession;
+ }
+
+ /**
+ * Transfers a single file
+ *
+ * @param source Source path in the source session
+ * @param destination Destination path in the destination session
+ * @param preserveAttributes Whether to preserve the attributes of the transferred file (e.g., permissions, file
+ * associated timestamps, etc.)
+ * @throws IOException If failed to transfer
+ */
+ public void transferFile(String source, String destination, boolean preserveAttributes) throws IOException {
+ Collection<Option> options = preserveAttributes
+ ? Collections.unmodifiableSet(EnumSet.of(Option.PreserveAttributes))
+ : Collections.emptySet();
+ String srcCmd = ScpClient.createReceiveCommand(source, options);
+ ClientSession srcSession = getSourceSession();
+ ClientSession dstSession = getDestinationSession();
+ boolean debugEnabled = log.isDebugEnabled();
+ if (debugEnabled) {
+ log.debug("transferFile({})[srcCmd='{}']) {} => {}",
+ this, srcCmd, source, destination);
+ }
+
+ ChannelExec srcChannel = ScpIoUtils.openCommandChannel(srcSession, srcCmd, log);
+ try (InputStream srcIn = srcChannel.getInvertedOut();
+ OutputStream srcOut = srcChannel.getInvertedIn()) {
+ String dstCmd = ScpClient.createSendCommand(destination, options);
+ if (debugEnabled) {
+ log.debug("transferFile({})[dstCmd='{}'} {} => {}",
+ this, dstCmd, source, destination);
+ }
+
+ ChannelExec dstChannel = ScpIoUtils.openCommandChannel(dstSession, dstCmd, log);
+ try (InputStream dstIn = dstChannel.getInvertedOut();
+ OutputStream dstOut = dstChannel.getInvertedIn()) {
+ redirectReceivedFile(source, srcIn, srcOut, destination, dstIn, dstOut);
+ } finally {
+ dstChannel.close(false);
+ }
+ } finally {
+ srcChannel.close(false);
+ }
+ }
+
+ protected long redirectReceivedFile(
+ String source, InputStream srcIn, OutputStream srcOut,
+ String destination, InputStream dstIn, OutputStream dstOut)
+ throws IOException {
+ int statusCode = transferStatusCode("XFER-FILE", dstIn, srcOut);
+ ScpIoUtils.validateCommandStatusCode("XFER-FILE", "redirectReceivedFile", statusCode, false);
+
+ boolean debugEnabled = log.isDebugEnabled();
+ String header = ScpIoUtils.readLine(srcIn, false);
+ if (debugEnabled) {
+ log.debug("redirectReceivedFile({}) header={}", this, header);
+ }
+
+ char cmdName = header.charAt(0);
+ ScpTimestamp time = null;
+ if (cmdName == ScpTimestamp.COMMAND_NAME) {
+ // Pass along the "T<mtime> 0 <atime> 0" and wait for response
+ time = ScpTimestamp.parseTime(header);
+ signalReceivedCommand(time);
+
+ ScpIoUtils.writeLine(dstOut, header);
+ statusCode = transferStatusCode(header, dstIn, srcOut);
+ ScpIoUtils.validateCommandStatusCode("[DST] " + header, "redirectReceivedFile", statusCode, false);
+
+ // Read the next command - which must be a 'C' command
+ header = ScpIoUtils.readLine(srcIn, false);
+ if (debugEnabled) {
+ log.debug("redirectReceivedFile({}) header={}", this, header);
+ }
+
+ cmdName = header.charAt(0);
+ }
+
+ if (cmdName != ScpReceiveFileCommandDetails.COMMAND_NAME) {
+ throw new StreamCorruptedException("Unexpected file command: " + header);
+ }
+
+ ScpReceiveFileCommandDetails details = new ScpReceiveFileCommandDetails(header);
+ signalReceivedCommand(details);
+
+ // Pass along the "Cmmmm <length> <filename" command and wait for ACK
+ ScpIoUtils.writeLine(dstOut, header);
+ statusCode = transferStatusCode(header, dstIn, srcOut);
+ ScpIoUtils.validateCommandStatusCode("[DST] " + header, "redirectReceivedFile", statusCode, false);
+ // Wait with ACK ready for transfer until ready to transfer data
+ long xferCount = transferFileData(source, srcIn, srcOut, destination, dstIn, dstOut, time, details);
+
+ // wait for source to signal data finished and pass it along
+ statusCode = transferStatusCode("SRC-EOF", srcIn, dstOut);
+ ScpIoUtils.validateCommandStatusCode("[SRC-EOF] " + header, "redirectReceivedFile", statusCode, false);
+
+ // wait for destination to signal data received
+ statusCode = ScpIoUtils.readAck(dstIn, false, log, "DST-EOF");
+ ScpIoUtils.validateCommandStatusCode("[DST-EOF] " + header, "redirectReceivedFile", statusCode, false);
+ return xferCount;
+ }
+
+ protected int transferStatusCode(Object logHint, InputStream in, OutputStream out) throws IOException {
+ int statusCode = in.read();
+ if (statusCode == -1) {
+ throw new EOFException("readAck(" + logHint + ") - EOF before ACK");
+ }
+
+ if (statusCode != ScpIoUtils.OK) {
+ String line = ScpIoUtils.readLine(in);
+ if (log.isDebugEnabled()) {
+ log.debug("transferStatusCode({})[{}] status={}, line='{}'", this, logHint, statusCode, line);
+ }
+ out.write(statusCode);
+ ScpIoUtils.writeLine(out, line);
+ } else {
+ if (log.isDebugEnabled()) {
+ log.debug("transferStatusCode({})[{}] status={}", this, logHint, statusCode);
+ }
+ out.write(statusCode);
+ out.flush();
+ }
+
+ return statusCode;
+ }
+
+ protected long transferFileData(
+ String source, InputStream srcIn, OutputStream srcOut,
+ String destination, InputStream dstIn, OutputStream dstOut,
+ ScpTimestamp time, ScpReceiveFileCommandDetails details)
+ throws IOException {
+ long length = details.getLength();
+ if (length < 0L) { // TODO consider throwing an exception...
+ log.warn("transferFileData({})[{} => {}] bad length in header: {}",
+ this, source, destination, details.toHeader());
+ }
+
+ ClientSession srcSession = getSourceSession();
+ ClientSession dstSession = getDestinationSession();
+ if (listener != null) {
+ listener.startDirectFileTransfer(srcSession, source, dstSession, destination, time, details);
+ }
+
+ long xferCount;
+ try (InputStream inputStream = new LimitInputStream(srcIn, length)) {
+ ScpIoUtils.ack(srcOut); // ready to receive the data from source
+ xferCount = IoUtils.copy(inputStream, dstOut);
+ dstOut.flush(); // make sure all data sent to destination
+ } catch (IOException | RuntimeException | Error e) {
+ if (listener != null) {
+ listener.endDirectFileTransfer(srcSession, source, dstSession, destination, time, details, 0L, e);
+ }
+ throw e;
+ }
+
+ if (log.isDebugEnabled()) {
+ log.debug("transferFileData({})[{} => {}] xfer {}/{} for {}",
+ this, source, destination, xferCount, length, details.getName());
+ }
+ if (listener != null) {
+ listener.endDirectFileTransfer(srcSession, source, dstSession, destination, time, details, xferCount, null);
+ }
+
+ return xferCount;
+ }
+
+ // Useful "hook" for implementors
+ protected void signalReceivedCommand(AbstractScpCommandDetails details) throws IOException {
+ if (log.isDebugEnabled()) {
+ log.debug("signalReceivedCommand({}) {}", this, details.toHeader());
+ }
+ }
+
+ @Override
+ public String toString() {
+ return getClass().getSimpleName() + "[src=" + getSourceSession() + ",dst=" + getDestinationSession() + "]";
+ }
+}
diff --git a/sshd-scp/src/main/java/org/apache/sshd/scp/client/ScpRemote2RemoteTransferListener.java b/sshd-scp/src/main/java/org/apache/sshd/scp/client/ScpRemote2RemoteTransferListener.java
new file mode 100644
index 0000000..1322495
--- /dev/null
+++ b/sshd-scp/src/main/java/org/apache/sshd/scp/client/ScpRemote2RemoteTransferListener.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sshd.scp.client;
+
+import java.io.IOException;
+
+import org.apache.sshd.client.session.ClientSession;
+import org.apache.sshd.scp.common.ScpTimestamp;
+import org.apache.sshd.scp.common.helpers.ScpReceiveFileCommandDetails;
+
+/**
+ * @author <a href="mailto:dev@mina.apache.org">Apache MINA SSHD Project</a>
+ */
+public interface ScpRemote2RemoteTransferListener {
+ /**
+ * Indicates start of direct file transfer
+ *
+ * @param srcSession The source {@link ClientSession}
+ * @param source The source path
+ * @param dstSession The destination {@link ClientSession}
+ * @param destination The destination path
+ * @param timestamp The {@link ScpTimestamp timestamp} of the file - may be {@code null}
+ * @param details The {@link ScpReceiveFileCommandDetails details} of the attempted file transfer
+ * @throws IOException If failed to handle the callback
+ */
+ void startDirectFileTransfer(
+ ClientSession srcSession, String source,
+ ClientSession dstSession, String destination,
+ ScpTimestamp timestamp, ScpReceiveFileCommandDetails details)
+ throws IOException;
+
+ /**
+ * Indicates end of direct file transfer
+ *
+ * @param srcSession The source {@link ClientSession}
+ * @param source The source path
+ * @param dstSession The destination {@link ClientSession}
+ * @param destination The destination path
+ * @param timestamp The {@link ScpTimestamp timestamp} of the file - may be {@code null}
+ * @param details The {@link ScpReceiveFileCommandDetails details} of the attempted file transfer
+ * @param xferSize Number of successfully transfered bytes - zero if <tt>thrown</tt> not {@code null}
+ * @param thrown Error thrown during transfer attempt - {@code null} if successful
+ * @throws IOException If failed to handle the callback
+ */
+ void endDirectFileTransfer(
+ ClientSession srcSession, String source,
+ ClientSession dstSession, String destination,
+ ScpTimestamp timestamp, ScpReceiveFileCommandDetails details,
+ long xferSize, Throwable thrown)
+ throws IOException;
+}
diff --git a/sshd-scp/src/main/java/org/apache/sshd/scp/server/ScpCommand.java b/sshd-scp/src/main/java/org/apache/sshd/scp/server/ScpCommand.java
index fcaf577..4cae4e7 100644
--- a/sshd-scp/src/main/java/org/apache/sshd/scp/server/ScpCommand.java
+++ b/sshd-scp/src/main/java/org/apache/sshd/scp/server/ScpCommand.java
@@ -190,7 +190,7 @@ public class ScpCommand extends AbstractFileSystemCommand {
writeCommandResponseMessage(command, exitValue, exitMessage);
} catch (IOException e2) {
log.error("run({})[{}] Failed ({}) to send error response: {}",
- session, command, e.getClass().getSimpleName(), e.getMessage());
+ session, command, e2.getClass().getSimpleName(), e2.getMessage());
if (debugEnabled) {
log.error("run(" + session + ")[" + command + "] error response failure details", e2);
}
diff --git a/sshd-scp/src/test/java/org/apache/sshd/scp/client/AbstractScpTestSupport.java b/sshd-scp/src/test/java/org/apache/sshd/scp/client/AbstractScpTestSupport.java
new file mode 100644
index 0000000..eb058b2
--- /dev/null
+++ b/sshd-scp/src/test/java/org/apache/sshd/scp/client/AbstractScpTestSupport.java
@@ -0,0 +1,158 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sshd.scp.client;
+
+import java.nio.file.Path;
+import java.nio.file.attribute.PosixFilePermission;
+import java.util.Collection;
+import java.util.Set;
+
+import org.apache.sshd.client.SshClient;
+import org.apache.sshd.client.session.ClientSession;
+import org.apache.sshd.common.config.keys.KeyUtils;
+import org.apache.sshd.common.file.FileSystemFactory;
+import org.apache.sshd.common.file.virtualfs.VirtualFileSystemFactory;
+import org.apache.sshd.common.session.Session;
+import org.apache.sshd.scp.common.ScpTransferEventListener;
+import org.apache.sshd.scp.server.ScpCommandFactory;
+import org.apache.sshd.server.SshServer;
+import org.apache.sshd.server.keyprovider.SimpleGeneratorHostKeyProvider;
+import org.apache.sshd.util.test.BaseTestSupport;
+import org.apache.sshd.util.test.CommonTestSupportUtils;
+import org.apache.sshd.util.test.CoreTestSupportUtils;
+import org.junit.AfterClass;
+import org.junit.Before;
+
+/**
+ * @author <a href="mailto:dev@mina.apache.org">Apache MINA SSHD Project</a>
+ */
+public abstract class AbstractScpTestSupport extends BaseTestSupport {
+ protected static final ScpTransferEventListener DEBUG_LISTENER = new ScpTransferEventListener() {
+ @Override
+ public void startFolderEvent(
+ Session s, FileOperation op, Path file, Set<PosixFilePermission> perms) {
+ logEvent("starFolderEvent", s, op, file, false, -1L, perms, null);
+ }
+
+ @Override
+ public void startFileEvent(
+ Session s, FileOperation op, Path file, long length, Set<PosixFilePermission> perms) {
+ logEvent("startFileEvent", s, op, file, true, length, perms, null);
+ }
+
+ @Override
+ public void endFolderEvent(
+ Session s, FileOperation op, Path file, Set<PosixFilePermission> perms, Throwable thrown) {
+ logEvent("endFolderEvent", s, op, file, false, -1L, perms, thrown);
+ }
+
+ @Override
+ public void endFileEvent(
+ Session s, FileOperation op, Path file, long length, Set<PosixFilePermission> perms, Throwable thrown) {
+ logEvent("endFileEvent", s, op, file, true, length, perms, thrown);
+ }
+
+ private void logEvent(
+ String type, Session s, FileOperation op, Path path, boolean isFile,
+ long length, Collection<PosixFilePermission> perms, Throwable t) {
+ if (!OUTPUT_DEBUG_MESSAGES) {
+ return; // just in case
+ }
+ StringBuilder sb = new StringBuilder(Byte.MAX_VALUE);
+ sb.append(" ").append(type)
+ .append('[').append(s).append(']')
+ .append('[').append(op).append(']')
+ .append(' ').append(isFile ? "File" : "Directory").append('=').append(path)
+ .append(' ').append("length=").append(length)
+ .append(' ').append("perms=").append(perms);
+ if (t != null) {
+ sb.append(' ').append("ERROR=").append(t.getClass().getSimpleName()).append(": ").append(t.getMessage());
+ }
+ outputDebugMessage(sb.toString());
+ }
+ };
+
+ protected static SshServer sshd;
+ protected static int port;
+ protected static SshClient client;
+
+ protected final FileSystemFactory fileSystemFactory;
+
+ protected AbstractScpTestSupport() {
+ Path targetPath = detectTargetFolder();
+ Path parentPath = targetPath.getParent();
+ fileSystemFactory = new VirtualFileSystemFactory(parentPath);
+ }
+
+ protected static void setupClientAndServer(Class<?> anchor) throws Exception {
+ // Need to use RSA since Ganymede/Jsch does not support EC
+ SimpleGeneratorHostKeyProvider provider = new SimpleGeneratorHostKeyProvider();
+ provider.setAlgorithm(KeyUtils.RSA_ALGORITHM);
+ provider.setKeySize(1024);
+
+ Path targetDir = CommonTestSupportUtils.detectTargetFolder(anchor);
+ provider.setPath(targetDir.resolve(anchor.getSimpleName() + "-key"));
+ sshd = CoreTestSupportUtils.setupTestFullSupportServer(anchor);
+ sshd.setKeyPairProvider(provider);
+
+ ScpCommandFactory factory = new ScpCommandFactory();
+ sshd.setCommandFactory(factory);
+ sshd.setShellFactory(factory);
+ sshd.start();
+ port = sshd.getPort();
+
+ client = CoreTestSupportUtils.setupTestFullSupportClient(anchor);
+ client.start();
+ }
+
+ @AfterClass
+ public static void tearDownClientAndServer() throws Exception {
+ if (sshd != null) {
+ try {
+ sshd.stop(true);
+ } finally {
+ sshd = null;
+ }
+ }
+
+ if (client != null) {
+ try {
+ client.stop();
+ } finally {
+ client = null;
+ }
+ }
+ }
+
+ protected static ScpTransferEventListener getScpTransferEventListener(ClientSession session) {
+ return OUTPUT_DEBUG_MESSAGES ? DEBUG_LISTENER : ScpTransferEventListener.EMPTY;
+ }
+
+ protected static ScpClient createScpClient(ClientSession session) {
+ ScpClientCreator creator = ScpClientCreator.instance();
+ ScpTransferEventListener listener = getScpTransferEventListener(session);
+ return creator.createScpClient(session, listener);
+ }
+
+ @Before
+ public void setUp() throws Exception {
+ sshd.setFileSystemFactory(fileSystemFactory);
+ }
+}
diff --git a/sshd-scp/src/test/java/org/apache/sshd/scp/client/ScpRemote2RemoteTransferHelperTest.java b/sshd-scp/src/test/java/org/apache/sshd/scp/client/ScpRemote2RemoteTransferHelperTest.java
new file mode 100644
index 0000000..7c0508b
--- /dev/null
+++ b/sshd-scp/src/test/java/org/apache/sshd/scp/client/ScpRemote2RemoteTransferHelperTest.java
@@ -0,0 +1,124 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sshd.scp.client;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.concurrent.atomic.AtomicLong;
+
+import org.apache.sshd.client.session.ClientSession;
+import org.apache.sshd.common.util.io.IoUtils;
+import org.apache.sshd.scp.common.ScpHelper;
+import org.apache.sshd.scp.common.ScpTimestamp;
+import org.apache.sshd.scp.common.helpers.ScpReceiveFileCommandDetails;
+import org.apache.sshd.util.test.CommonTestSupportUtils;
+import org.junit.BeforeClass;
+import org.junit.FixMethodOrder;
+import org.junit.Test;
+import org.junit.runners.MethodSorters;
+
+/**
+ * @author <a href="mailto:dev@mina.apache.org">Apache MINA SSHD Project</a>
+ */
+@FixMethodOrder(MethodSorters.NAME_ASCENDING)
+public class ScpRemote2RemoteTransferHelperTest extends AbstractScpTestSupport {
+ public ScpRemote2RemoteTransferHelperTest() {
+ super();
+ }
+
+ @BeforeClass
+ public static void setupClientAndServer() throws Exception {
+ setupClientAndServer(ScpRemote2RemoteTransferHelperTest.class);
+ }
+
+ @Test
+ public void testTransferFiles() throws Exception {
+ Path targetPath = detectTargetFolder();
+ Path parentPath = targetPath.getParent();
+ Path scpRoot = CommonTestSupportUtils.resolve(targetPath,
+ ScpHelper.SCP_COMMAND_PREFIX, getClass().getSimpleName(), getCurrentTestName());
+ CommonTestSupportUtils.deleteRecursive(scpRoot); // start clean
+
+ Path srcDir = assertHierarchyTargetFolderExists(scpRoot.resolve("srcdir"));
+ Path srcFile = srcDir.resolve("source.txt");
+ byte[] expectedData
+ = CommonTestSupportUtils.writeFile(srcFile, getClass().getName() + "#" + getCurrentTestName() + IoUtils.EOL);
+ String srcPath = CommonTestSupportUtils.resolveRelativeRemotePath(parentPath, srcFile);
+
+ Path dstDir = assertHierarchyTargetFolderExists(scpRoot.resolve("dstdir"));
+ Path dstFile = dstDir.resolve("destination.txt");
+ String dstPath = CommonTestSupportUtils.resolveRelativeRemotePath(parentPath, dstFile);
+
+ AtomicLong xferCount = new AtomicLong();
+ try (ClientSession srcSession = createClientSession(getCurrentTestName() + "-src");
+ ClientSession dstSession = createClientSession(getCurrentTestName() + "-dst")) {
+ ScpRemote2RemoteTransferHelper helper = new ScpRemote2RemoteTransferHelper(
+ srcSession, dstSession, new ScpRemote2RemoteTransferListener() {
+ @Override
+ public void startDirectFileTransfer(
+ ClientSession srcSession, String source,
+ ClientSession dstSession, String destination,
+ ScpTimestamp timestamp, ScpReceiveFileCommandDetails details)
+ throws IOException {
+ assertEquals("Mismatched start xfer source path", srcPath, source);
+ assertEquals("Mismatched start xfer destination path", dstPath, destination);
+ }
+
+ @Override
+ public void endDirectFileTransfer(
+ ClientSession srcSession, String source,
+ ClientSession dstSession, String destination,
+ ScpTimestamp timestamp, ScpReceiveFileCommandDetails details,
+ long xferSize, Throwable thrown)
+ throws IOException {
+ assertEquals("Mismatched end xfer source path", srcPath, source);
+ assertEquals("Mismatched end xfer destination path", dstPath, destination);
+
+ long prev = xferCount.getAndSet(xferSize);
+ assertEquals("Mismatched 1st end file xfer size", 0L, prev);
+ }
+ });
+ helper.transferFile(srcPath, dstPath, true);
+ }
+ assertEquals("Mismatched transfer size", expectedData.length, xferCount.getAndSet(0L));
+
+ byte[] actualData = Files.readAllBytes(dstFile);
+ assertArrayEquals("Mismatched transfer contents", expectedData, actualData);
+ }
+
+ private ClientSession createClientSession(String username) throws IOException {
+ ClientSession session = client.connect(username, TEST_LOCALHOST, port)
+ .verify(CONNECT_TIMEOUT)
+ .getSession();
+ try {
+ session.addPasswordIdentity(username);
+ session.auth().verify(AUTH_TIMEOUT);
+
+ ClientSession result = session;
+ session = null; // avoid auto-close at finally clause
+ return result;
+ } finally {
+ if (session != null) {
+ session.close();
+ }
+ }
+ }
+}
diff --git a/sshd-scp/src/test/java/org/apache/sshd/scp/client/ScpTest.java b/sshd-scp/src/test/java/org/apache/sshd/scp/client/ScpTest.java
index 788c0fa..0d5583b 100644
--- a/sshd-scp/src/test/java/org/apache/sshd/scp/client/ScpTest.java
+++ b/sshd-scp/src/test/java/org/apache/sshd/scp/client/ScpTest.java
@@ -18,7 +18,6 @@
*/
package org.apache.sshd.scp.client;
-import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
@@ -40,12 +39,15 @@ import java.util.Set;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
-import org.apache.sshd.client.SshClient;
+import ch.ethz.ssh2.Connection;
+import ch.ethz.ssh2.ConnectionInfo;
+import ch.ethz.ssh2.SCPClient;
+import com.jcraft.jsch.ChannelExec;
+import com.jcraft.jsch.JSch;
+import com.jcraft.jsch.JSchException;
import org.apache.sshd.client.session.ClientSession;
import org.apache.sshd.common.Factory;
import org.apache.sshd.common.channel.Channel;
-import org.apache.sshd.common.config.keys.KeyUtils;
-import org.apache.sshd.common.file.FileSystemFactory;
import org.apache.sshd.common.file.virtualfs.VirtualFileSystemFactory;
import org.apache.sshd.common.io.BuiltinIoServiceFactoryFactories;
import org.apache.sshd.common.random.Random;
@@ -67,91 +69,25 @@ import org.apache.sshd.scp.common.helpers.ScpReceiveDirCommandDetails;
import org.apache.sshd.scp.common.helpers.ScpReceiveFileCommandDetails;
import org.apache.sshd.scp.server.ScpCommand;
import org.apache.sshd.scp.server.ScpCommandFactory;
-import org.apache.sshd.server.SshServer;
import org.apache.sshd.server.channel.ChannelSession;
import org.apache.sshd.server.command.Command;
-import org.apache.sshd.server.keyprovider.SimpleGeneratorHostKeyProvider;
-import org.apache.sshd.util.test.BaseTestSupport;
import org.apache.sshd.util.test.CommonTestSupportUtils;
-import org.apache.sshd.util.test.CoreTestSupportUtils;
import org.apache.sshd.util.test.JSchLogger;
import org.apache.sshd.util.test.SimpleUserInfo;
-import org.junit.AfterClass;
-import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.FixMethodOrder;
import org.junit.Test;
import org.junit.runners.MethodSorters;
-import com.jcraft.jsch.ChannelExec;
-import com.jcraft.jsch.JSch;
-import com.jcraft.jsch.JSchException;
-
-import ch.ethz.ssh2.Connection;
-import ch.ethz.ssh2.ConnectionInfo;
-import ch.ethz.ssh2.SCPClient;
-
/**
* Test for SCP support.
*
* @author <a href="mailto:dev@mina.apache.org">Apache MINA SSHD Project</a>
*/
@FixMethodOrder(MethodSorters.NAME_ASCENDING)
-public class ScpTest extends BaseTestSupport {
- private static final ScpTransferEventListener DEBUG_LISTENER = new ScpTransferEventListener() {
- @Override
- public void startFolderEvent(
- Session s, FileOperation op, Path file, Set<PosixFilePermission> perms) {
- logEvent("starFolderEvent", s, op, file, false, -1L, perms, null);
- }
-
- @Override
- public void startFileEvent(
- Session s, FileOperation op, Path file, long length, Set<PosixFilePermission> perms) {
- logEvent("startFileEvent", s, op, file, true, length, perms, null);
- }
-
- @Override
- public void endFolderEvent(
- Session s, FileOperation op, Path file, Set<PosixFilePermission> perms, Throwable thrown) {
- logEvent("endFolderEvent", s, op, file, false, -1L, perms, thrown);
- }
-
- @Override
- public void endFileEvent(
- Session s, FileOperation op, Path file, long length, Set<PosixFilePermission> perms, Throwable thrown) {
- logEvent("endFileEvent", s, op, file, true, length, perms, thrown);
- }
-
- private void logEvent(
- String type, Session s, FileOperation op, Path path, boolean isFile,
- long length, Collection<PosixFilePermission> perms, Throwable t) {
- if (!OUTPUT_DEBUG_MESSAGES) {
- return; // just in case
- }
- StringBuilder sb = new StringBuilder(Byte.MAX_VALUE);
- sb.append(" ").append(type)
- .append('[').append(s).append(']')
- .append('[').append(op).append(']')
- .append(' ').append(isFile ? "File" : "Directory").append('=').append(path)
- .append(' ').append("length=").append(length)
- .append(' ').append("perms=").append(perms);
- if (t != null) {
- sb.append(' ').append("ERROR=").append(t.getClass().getSimpleName()).append(": ").append(t.getMessage());
- }
- outputDebugMessage(sb.toString());
- }
- };
-
- private static SshServer sshd;
- private static int port;
- private static SshClient client;
- private final FileSystemFactory fileSystemFactory;
-
+public class ScpTest extends AbstractScpTestSupport {
public ScpTest() throws IOException {
- Path targetPath = detectTargetFolder();
- Path parentPath = targetPath.getParent();
- fileSystemFactory = new VirtualFileSystemFactory(parentPath);
+ super();
}
@BeforeClass
@@ -160,51 +96,6 @@ public class ScpTest extends BaseTestSupport {
setupClientAndServer(ScpTest.class);
}
- protected static void setupClientAndServer(Class<?> anchor) throws Exception {
- // Need to use RSA since Ganymede does not support EC
- SimpleGeneratorHostKeyProvider provider = new SimpleGeneratorHostKeyProvider();
- provider.setAlgorithm(KeyUtils.RSA_ALGORITHM);
- provider.setKeySize(1024);
-
- Path targetDir = CommonTestSupportUtils.detectTargetFolder(anchor);
- provider.setPath(targetDir.resolve(anchor.getSimpleName() + "-key"));
- sshd = CoreTestSupportUtils.setupTestFullSupportServer(anchor);
- sshd.setKeyPairProvider(provider);
-
- ScpCommandFactory factory = new ScpCommandFactory();
- sshd.setCommandFactory(factory);
- sshd.setShellFactory(factory);
- sshd.start();
- port = sshd.getPort();
-
- client = CoreTestSupportUtils.setupTestFullSupportClient(anchor);
- client.start();
- }
-
- @AfterClass
- public static void tearDownClientAndServer() throws Exception {
- if (sshd != null) {
- try {
- sshd.stop(true);
- } finally {
- sshd = null;
- }
- }
-
- if (client != null) {
- try {
- client.stop();
- } finally {
- client = null;
- }
- }
- }
-
- @Before
- public void setUp() throws Exception {
- sshd.setFileSystemFactory(fileSystemFactory);
- }
-
@Test
public void testNormalizedScpRemotePaths() throws Exception {
// see SSHD-822
@@ -1144,7 +1035,7 @@ public class ScpTest extends BaseTestSupport {
os.write(0);
os.flush();
- String header = readLine(is);
+ String header = ScpIoUtils.readLine(is, false);
String expHeader
= ScpReceiveFileCommandDetails.COMMAND_NAME + ScpReceiveFileCommandDetails.DEFAULT_FILE_OCTAL_PERMISSIONS
+ " " + Files.size(target) + " " + fileName;
@@ -1176,38 +1067,33 @@ public class ScpTest extends BaseTestSupport {
try (OutputStream os = c.getOutputStream();
InputStream is = c.getInputStream()) {
- os.write(0);
- os.flush();
+ ScpIoUtils.ack(os);
- String header = readLine(is);
+ String header = ScpIoUtils.readLine(is, false);
String expPrefix = ScpReceiveDirCommandDetails.COMMAND_NAME
+ ScpReceiveDirCommandDetails.DEFAULT_DIR_OCTAL_PERMISSIONS + " 0 ";
assertTrue("Bad header prefix for " + path + ": " + header, header.startsWith(expPrefix));
- os.write(0);
- os.flush();
+ ScpIoUtils.ack(os);
- header = readLine(is);
+ header = ScpIoUtils.readLine(is, false);
String fileName = Objects.toString(target.getFileName(), null);
String expHeader
= ScpReceiveFileCommandDetails.COMMAND_NAME + ScpReceiveFileCommandDetails.DEFAULT_FILE_OCTAL_PERMISSIONS
+ " " + Files.size(target) + " " + fileName;
assertEquals("Mismatched dir header for " + path, expHeader, header);
int length = Integer.parseInt(header.substring(6, header.indexOf(' ', 6)));
- os.write(0);
- os.flush();
+ ScpIoUtils.ack(os);
byte[] buffer = new byte[length];
length = is.read(buffer, 0, buffer.length);
assertEquals("Mismatched read buffer size for " + path, length, buffer.length);
assertAckReceived(is, "Read date of " + path);
- os.write(0);
- os.flush();
+ ScpIoUtils.ack(os);
- header = readLine(is);
+ header = ScpIoUtils.readLine(is, false);
assertEquals("Mismatched end value for " + path, "E", header);
- os.write(0);
- os.flush();
+ ScpIoUtils.ack(os);
return new String(buffer, StandardCharsets.UTF_8);
} finally {
@@ -1224,9 +1110,8 @@ public class ScpTest extends BaseTestSupport {
try (OutputStream os = c.getOutputStream();
InputStream is = c.getInputStream()) {
- os.write(0);
- os.flush();
- assertEquals("Mismatched response for command: " + command, 2, is.read());
+ ScpIoUtils.ack(os);
+ assertEquals("Mismatched response for command: " + command, ScpIoUtils.ERROR, is.read());
} finally {
c.disconnect();
}
@@ -1254,8 +1139,7 @@ public class ScpTest extends BaseTestSupport {
os.flush();
assertAckReceived(is, "Sent data (length=" + data.length() + ") for " + path + "[" + name + "]");
- os.write(0);
- os.flush();
+ ScpIoUtils.ack(os);
Thread.sleep(100);
} finally {
@@ -1264,8 +1148,7 @@ public class ScpTest extends BaseTestSupport {
}
protected void assertAckReceived(OutputStream os, InputStream is, String command) throws IOException {
- os.write((command + "\n").getBytes(StandardCharsets.UTF_8));
- os.flush();
+ ScpIoUtils.writeLine(os, command);
assertAckReceived(is, command);
}
@@ -1285,9 +1168,8 @@ public class ScpTest extends BaseTestSupport {
assertAckReceived(is, command);
command = "C7777 " + data.length() + " " + name;
- os.write((command + "\n").getBytes(StandardCharsets.UTF_8));
- os.flush();
- assertEquals("Mismatched response for command=" + command, 2, is.read());
+ ScpIoUtils.writeLine(os, command);
+ assertEquals("Mismatched response for command=" + command, ScpIoUtils.ERROR, is.read());
} finally {
c.disconnect();
}
@@ -1313,35 +1195,9 @@ public class ScpTest extends BaseTestSupport {
ScpIoUtils.ack(os);
ScpIoUtils.writeLine(os, ScpDirEndCommandDetails.HEADER);
-
assertAckReceived(is, "Signal end of " + path);
} finally {
c.disconnect();
}
}
-
- private static String readLine(InputStream in) throws IOException {
- try (OutputStream baos = new ByteArrayOutputStream()) {
- for (;;) {
- int c = in.read();
- if (c == '\n') {
- return baos.toString();
- } else if (c == -1) {
- throw new IOException("End of stream");
- } else {
- baos.write(c);
- }
- }
- }
- }
-
- private static ScpClient createScpClient(ClientSession session) {
- ScpClientCreator creator = ScpClientCreator.instance();
- ScpTransferEventListener listener = getScpTransferEventListener(session);
- return creator.createScpClient(session, listener);
- }
-
- private static ScpTransferEventListener getScpTransferEventListener(ClientSession session) {
- return OUTPUT_DEBUG_MESSAGES ? DEBUG_LISTENER : ScpTransferEventListener.EMPTY;
- }
}