You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airavata.apache.org by di...@apache.org on 2019/12/11 07:34:28 UTC
[airavata-mft] branch master updated: Implementing a new streaming
buffer and local transport
This is an automated email from the ASF dual-hosted git repository.
dimuthuupe pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/airavata-mft.git
The following commit(s) were added to refs/heads/master by this push:
new ab9856b Implementing a new streaming buffer and local transport
ab9856b is described below
commit ab9856b3102f3f4929856e78ef20acd646f4da1f
Author: Dimuthu Wannipurage <di...@gmail.com>
AuthorDate: Wed Dec 11 02:34:14 2019 -0500
Implementing a new streaming buffer and local transport
---
agent/pom.xml | 5 +
.../org/apache/airavata/mft/agent/MFTAgent.java | 16 +-
.../airavata/mft/agent/TransportMediator.java | 5 +-
.../airavata/mft/core/CircularStreamingBuffer.java | 116 ++++
.../apache/airavata/mft/core/ConnectorContext.java | 6 +-
.../org/apache/airavata/mft/core/StreamBuffer.java | 604 ---------------------
.../mft/core/CircularStreamingBufferTest.java | 265 +++++++++
transport/local-transport/pom.xml | 22 +
.../transport/local/LocalMetadataCollector.java | 20 +
.../mft/transport/local/LocalReceiver.java | 59 ++
.../transport/local/LocalResourceIdentifier.java | 14 +
.../airavata/mft/transport/local/LocalSender.java | 54 ++
.../mft/transport/local/LocalTransportUtil.java | 18 +
transport/pom.xml | 1 +
.../airavata/mft/transport/scp/SCPReceiver.java | 4 +-
.../airavata/mft/transport/scp/SCPSender.java | 5 +-
16 files changed, 599 insertions(+), 615 deletions(-)
diff --git a/agent/pom.xml b/agent/pom.xml
index 8559b82..a8846f5 100644
--- a/agent/pom.xml
+++ b/agent/pom.xml
@@ -38,5 +38,10 @@
<artifactId>mft-scp-transport</artifactId>
<version>0.01-SNAPSHOT</version>
</dependency>
+ <dependency>
+ <groupId>org.apache.airavata</groupId>
+ <artifactId>mft-local-transport</artifactId>
+ <version>0.01-SNAPSHOT</version>
+ </dependency>
</dependencies>
</project>
\ No newline at end of file
diff --git a/agent/src/main/java/org/apache/airavata/mft/agent/MFTAgent.java b/agent/src/main/java/org/apache/airavata/mft/agent/MFTAgent.java
index de4fe72..1b050f0 100644
--- a/agent/src/main/java/org/apache/airavata/mft/agent/MFTAgent.java
+++ b/agent/src/main/java/org/apache/airavata/mft/agent/MFTAgent.java
@@ -3,6 +3,9 @@ package org.apache.airavata.mft.agent;
import org.apache.airavata.mft.core.ResourceMetadata;
import org.apache.airavata.mft.core.api.Connector;
import org.apache.airavata.mft.core.api.MetadataCollector;
+import org.apache.airavata.mft.transport.local.LocalMetadataCollector;
+import org.apache.airavata.mft.transport.local.LocalReceiver;
+import org.apache.airavata.mft.transport.local.LocalSender;
import org.apache.airavata.mft.transport.scp.SCPMetadataCollector;
import org.apache.airavata.mft.transport.scp.SCPReceiver;
import org.apache.airavata.mft.transport.scp.SCPSender;
@@ -26,7 +29,7 @@ public class MFTAgent {
MetadataCollector metadataCollector = resolveMetadataCollector(request.getSourceType());
ResourceMetadata metadata = metadataCollector.getGetResourceMetadata(request.getSourceId(), request.getSourceToken());
-
+ System.out.println("File size " + metadata.getResourceSize());
mediator.transfer(inConnector, outConnector, metadata);
} catch (Exception e) {
@@ -39,7 +42,7 @@ public class MFTAgent {
request.setSourceId("1");
request.setSourceType("SCP");
request.setDestinationId("2");
- request.setDestinationType("SCP");
+ request.setDestinationType("LOCAL");
MFTAgent agent = new MFTAgent();
agent.requests.add(request);
@@ -57,6 +60,13 @@ public class MFTAgent {
return new SCPSender();
}
break;
+ case "LOCAL":
+ switch (direction) {
+ case "IN":
+ return new LocalReceiver();
+ case "OUT":
+ return new LocalSender();
+ }
}
return null;
}
@@ -66,6 +76,8 @@ public class MFTAgent {
switch (type) {
case "SCP":
return new SCPMetadataCollector();
+ case "LOCAL":
+ return new LocalMetadataCollector();
}
return null;
}
diff --git a/agent/src/main/java/org/apache/airavata/mft/agent/TransportMediator.java b/agent/src/main/java/org/apache/airavata/mft/agent/TransportMediator.java
index 319ee4e..f20570d 100644
--- a/agent/src/main/java/org/apache/airavata/mft/agent/TransportMediator.java
+++ b/agent/src/main/java/org/apache/airavata/mft/agent/TransportMediator.java
@@ -1,8 +1,8 @@
package org.apache.airavata.mft.agent;
+import org.apache.airavata.mft.core.CircularStreamingBuffer;
import org.apache.airavata.mft.core.ConnectorContext;
import org.apache.airavata.mft.core.ResourceMetadata;
-import org.apache.airavata.mft.core.StreamBuffer;
import org.apache.airavata.mft.core.TransferTask;
import org.apache.airavata.mft.core.api.Connector;
@@ -20,7 +20,7 @@ public class TransportMediator {
public void transfer(Connector inConnector, Connector outConnector, ResourceMetadata metadata) throws Exception {
- StreamBuffer streamBuffer = new StreamBuffer();
+ CircularStreamingBuffer streamBuffer = new CircularStreamingBuffer();
ConnectorContext context = new ConnectorContext();
context.setMetadata(metadata);
context.setStreamBuffer(streamBuffer);
@@ -46,6 +46,7 @@ public class TransportMediator {
System.out.println("One task failed with error: " + e.getMessage() );
e.printStackTrace();
for(Future<Integer> f : futureList){
+ Thread.sleep(1000);
f.cancel(true);
}
futureList.clear();
diff --git a/core/src/main/java/org/apache/airavata/mft/core/CircularStreamingBuffer.java b/core/src/main/java/org/apache/airavata/mft/core/CircularStreamingBuffer.java
new file mode 100644
index 0000000..b18e0c3
--- /dev/null
+++ b/core/src/main/java/org/apache/airavata/mft/core/CircularStreamingBuffer.java
@@ -0,0 +1,116 @@
+package org.apache.airavata.mft.core;
+
+import java.io.*;
+import java.util.concurrent.ArrayBlockingQueue;
+import java.util.concurrent.Semaphore;
+
+public class CircularStreamingBuffer {
+
+ private int bufferSize = 1024;
+ private ArrayBlockingQueue<Byte> buffer = new ArrayBlockingQueue<>(bufferSize);
+
+ private boolean osClosed = false;
+ private Semaphore readSem = new Semaphore(0);
+
+ private OutputStream outputStream = new CSBOutputStream();
+ private InputStream inputStream = new CSBInputStream();
+
+ public OutputStream getOutputStream() {
+ return outputStream;
+ }
+
+ public InputStream getInputStream() {
+ return inputStream;
+ }
+
+ private void updateRead() {
+ if (readSem.availablePermits() == 0) {
+ readSem.release();
+ }
+ }
+
+ public class CSBOutputStream extends OutputStream {
+ @Override
+ public void write(byte[] b) throws IOException {
+ write(b, 0, b.length);
+ }
+
+ @Override
+ public void write(byte[] b, int off, int len) throws IOException {
+ for (int i = off; i < len; i ++) {
+ try {
+ buffer.put(b[i]);
+ updateRead();
+ } catch (InterruptedException e) {
+ throw new IOException(e);
+ }
+ }
+ }
+
+ @Override
+ public void flush() throws IOException {
+ }
+
+ @Override
+ public void close() throws IOException {
+ osClosed = true;
+ updateRead();
+ }
+
+ @Override
+ public void write(int b) throws IOException {
+ try {
+ buffer.put((byte)b);
+ //System.out.println("Writing");
+ updateRead();
+ } catch (InterruptedException e) {
+ throw new IOException(e);
+ }
+ }
+ }
+
+ public class CSBInputStream extends InputStream {
+
+ @Override
+ public int read() throws IOException {
+ Byte res = buffer.poll();
+ if (res == null) {
+ //System.out.println("Received null in is.read()");
+ if (osClosed) return -1;
+ try {
+ readSem.acquire();
+ return read();
+ } catch (InterruptedException e) {
+ throw new IOException(e);
+ }
+ } else {
+ return res & 0xff;
+ }
+ }
+
+ @Override
+ public int read(byte[] b, int off, int len) throws IOException {
+ for (int i = off; i < len; i++) {
+ int res = read();
+ if (res == -1) {
+ //System.out.println("Received -1 in is.read(byte[], off, len)");
+ if (i == off) {
+ //System.out.println("Return -1");
+ return -1;
+ } else {
+ //System.out.println("Return " + (i - off));
+ return i - off;
+ }
+ } else {
+ b[i] = (byte)res;
+ }
+ }
+ return len - off;
+ }
+
+ @Override
+ public int read(byte[] b) throws IOException {
+ return read(b, 0, b.length);
+ }
+ }
+}
\ No newline at end of file
diff --git a/core/src/main/java/org/apache/airavata/mft/core/ConnectorContext.java b/core/src/main/java/org/apache/airavata/mft/core/ConnectorContext.java
index 24485c5..2592108 100644
--- a/core/src/main/java/org/apache/airavata/mft/core/ConnectorContext.java
+++ b/core/src/main/java/org/apache/airavata/mft/core/ConnectorContext.java
@@ -2,14 +2,14 @@ package org.apache.airavata.mft.core;
public class ConnectorContext {
- private StreamBuffer streamBuffer;
+ private CircularStreamingBuffer streamBuffer;
private ResourceMetadata metadata;
- public StreamBuffer getStreamBuffer() {
+ public CircularStreamingBuffer getStreamBuffer() {
return streamBuffer;
}
- public void setStreamBuffer(StreamBuffer streamBuffer) {
+ public void setStreamBuffer(CircularStreamingBuffer streamBuffer) {
this.streamBuffer = streamBuffer;
}
diff --git a/core/src/main/java/org/apache/airavata/mft/core/StreamBuffer.java b/core/src/main/java/org/apache/airavata/mft/core/StreamBuffer.java
deleted file mode 100644
index 715d3ef..0000000
--- a/core/src/main/java/org/apache/airavata/mft/core/StreamBuffer.java
+++ /dev/null
@@ -1,604 +0,0 @@
-// @formatter:off
-/**
- * Copyright 2014 Bernard Ladenthin bernard.ladenthin@gmail.com
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-// @formatter:on
-package org.apache.airavata.mft.core;
-
-import java.io.Closeable;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.util.Deque;
-import java.util.LinkedList;
-import java.util.concurrent.Semaphore;
-
-// Copied from https://github.com/bernardladenthin/streambuffer
-/**
- * A stream buffer is a class to buffer data that has been written to an
- * {@link OutputStream} and provide the data in an {@link InputStream}. There is
- * no need to close the stream before read. The read works during a write
- * process. It is possible to call concurrent a method in the
- * {@link OutputStream} and {@link InputStream}. Read/Write at the same time.
- *
- * @author Bernard Ladenthin bernard.ladenthin@gmail.com
- */
-public class StreamBuffer implements Closeable {
-
- /**
- * An object to get an unique access to the {@link #buffer}. It is needed to
- * get an exclusive access for read and write operations.
- */
- private final Object bufferLock = new Object();
-
- /**
- * The buffer which contains the raw data.
- */
- private final Deque<byte[]> buffer = new LinkedList<>();
-
- /**
- * A {@link Semaphore} to signal that data are added to the {@link #buffer}.
- * This signal is also used to announce that the stream was closed.
- * This {@link Semaphore} is used only for internal communication.
- */
- private final Semaphore signalModification = new Semaphore(0);
-
- /**
- * A {@link Semaphore} to signal that data are added to the {@link #buffer}.
- * This signal is also used to announce that the stream was closed.
- * This {@link Semaphore} is used only for external communication.
- */
- private final Semaphore signalModificationExternal = new Semaphore(0);
-
- /**
- * A variable for the current position of the current element in the
- * {@link #buffer}.
- */
- private volatile int positionAtCurrentBufferEntry = 0;
-
- /**
- * The sum of available bytes.
- */
- private volatile long availableBytes = 0;
-
- /**
- * A flag to indicate the stream was closed.
- */
- private volatile boolean streamClosed = false;
-
- /**
- * A flag to enable a safe write. If safe write is enabled, modifiable byte
- * arrays are cloned before they are written (put) into the FIFO. Benefit:
- * It cost more performance to clone a byte array, but the content is
- * immutable. This may be revoked by {@link #ignoreSafeWrite}, for example a
- * byte array was created internally.
- */
- private volatile boolean safeWrite = false;
-
- /**
- * A flag to disable the {@link StreamBuffer#safeWrite}. I. e. to write a byte array from
- * the trim method.
- */
- private boolean ignoreSafeWrite = false;
-
- /**
- * The maximum buffer elements. A number of maximum elements to invoke the
- * trim method. If the buffer contains more elements as the maximum, the
- * buffer will be completely reduced to one element. This option can help to
- * improve the performance for a read of huge amount data. It can also help
- * to shrink the size of the memory used. Use a value lower or equals
- * <code>0</code> to disable the trim option. Default is a maximum of
- * <code>100</code> elements.
- */
- private volatile int maxBufferElements = 100;
-
- private final SBInputStream is = new SBInputStream();
- private final SBOutputStream os = new SBOutputStream();
-
- /**
- * Construct a new {@link StreamBuffer}.
- */
- public StreamBuffer() {
- }
-
- /**
- * Returns the flag which indicates whether write operations are executed
- * safe or unsafe.
- *
- * @return <code>true</code> if the safe write option is enabled, otherwise
- * <code>false</code>.
- */
- public boolean isSafeWrite() {
- return safeWrite;
- }
-
- /**
- * Set a secure or unsercure write operation.
- *
- * @param safeWrite A flag to enable a safe write. If safe write is enabled,
- * modifiable byte arrays are cloned before they are written. Benefit: It
- * cost more performance to clone a byte array, but it is hardened to
- * external changes. Use <code>true</code> to force a clone. Otherwise use
- * <code>false</code> (default).
- */
- public void setSafeWrite(boolean safeWrite) {
- this.safeWrite = safeWrite;
- }
-
- /**
- * Returns the number of maximum elements which are held in maximum memory.
- *
- * @return the number of elements.
- */
- public int getMaxBufferElements() {
- return maxBufferElements;
- }
-
- /**
- * Set maximum elements for the buffer. Change the value doesn't invoke a
- * trim call if the buffer contains more elements. Only write operations
- * force a trim call.
- *
- * @param maxBufferElements number of maximum elements.
- */
- public void setMaxBufferElements(int maxBufferElements) {
- this.maxBufferElements = maxBufferElements;
- }
-
- /**
- * Security check mostly copied from {@link InputStream#read(byte[], int, int)}.
- * Ensures the parameter are valid.
- * @param b the byte array to copy from
- * @param off the offset to read from the array
- * @param len the len of the bytes to read from the array
- * @return <code>true</code> if there are bytes to read, otherwise <code>false</code>
- * @throws NullPointerException if the array is null
- * @throws IndexOutOfBoundsException if the index is not correct
- */
- public static boolean correctOffsetAndLengthToRead(byte[] b, int off, int len) {
- if (b == null) {
- throw new NullPointerException();
- } else if (off < 0 || len < 0 || len > b.length - off) {
- throw new IndexOutOfBoundsException();
- } else if (len == 0) {
- return false;
- }
- return true;
- }
-
- /**
- * Security check mostly copied from {@link OutputStream#write(byte[], int, int)}.
- * Ensures the parameter are valid.
- * @param b the byte array to write to the array
- * @param off the offset to write to the array
- * @param len the len of the bytes to write to the array
- * @return <code>true</code> if there are bytes to write, otherwise <code>false</code>
- * @throws NullPointerException if the array is null
- * @throws IndexOutOfBoundsException if the index is not correct
- */
- public static boolean correctOffsetAndLengthToWrite(byte[] b, int off, int len) {
- if (b == null) {
- throw new NullPointerException();
- } else if ((off < 0) || (off > b.length) || (len < 0)
- || ((off + len) > b.length) || ((off + len) < 0)) {
- throw new IndexOutOfBoundsException();
- } else if (len == 0) {
- return false;
- }
- return true;
- }
-
- /**
- * Call this method always after all changes are synchronized. This method
- * signals a modification. It cloud be a write on the stream or a close.
- */
- private void signalModification() {
- // hold up the permits to a maximum of one
- if (signalModification.availablePermits() == 0) {
- signalModification.release();
- }
- // same twice for external communication
- if (signalModificationExternal.availablePermits() == 0) {
- signalModificationExternal.release();
- }
- }
-
- /**
- * This method is blocking until data is available on the
- * {@link InputStream} or the stream was closed. This method could be called
- * from one thread only. This method could not be used to notify two threads.
- * @throws java.lang.InterruptedException if the current thread is interrupted
- */
- public void blockDataAvailable() throws InterruptedException {
- if (isClosed()) {
- return;
- }
-
- if (availableBytes < 1) {
- signalModificationExternal.acquire();
- }
- }
-
- /**
- * This method trims the buffer. This method can be invoked after every
- * write operation. The method checks itself if the buffer should be trimmed
- * or not.
- */
- private void trim() throws IOException {
- if (isTrimShouldBeExecuted()) {
-
- /**
- * Need to store more bufs, may it is not possible to read out all
- * data at once. The available method only returns an int value
- * instead a long value. Store all read parts of the full buffer in
- * a deque.
- */
- final Deque<byte[]> tmpBuffer = new LinkedList<>();
-
- int available;
- // empty the current buffer, read out all bytes
- while ((available = is.available()) > 0) {
- final byte[] buf = new byte[available];
- // read out of the buffer
- // and store the result to the tmpBuffer
- int read = is.read(buf);
- // should never happen
- assert read == available : "Read not enough bytes from buffer.";
- tmpBuffer.add(buf);
- }
- /**
- * Write all previously read parts back to the buffer. The buffer is
- * clean and contains no elements because all parts are read out.
- */
- try {
- /**
- * The reference to the buffer deque is not reachable out of the
- * trim method. It is possible to ignore the safeWrite flag to
- * prevent not necessary clone operations.
- */
- ignoreSafeWrite = true;
- while (!tmpBuffer.isEmpty()) {
- // pollFirst returns always a non null value, tmpBuffer is only filled with non null values
- os.write(tmpBuffer.pollFirst());
- }
- } finally {
- ignoreSafeWrite = false;
- }
- }
- }
-
- /**
- * Checks if a trim should be performed.
- * @return <code>true</code> if a trim should be performed, otherwise <code>false</code>.
- */
- private boolean isTrimShouldBeExecuted() {
- /**
- * To be thread safe, cache the maxBufferElements value. May the method
- * {@link #setMaxBufferElements(int)} was invoked from outside by another thread.
- */
- final int maxBufferElements = getMaxBufferElements();
- return (maxBufferElements > 0) && (buffer.size() >= 2) && (buffer.size() > maxBufferElements);
- }
-
- /**
- * This method mustn't be called in a synchronized context, the variable is
- * volatile.
- */
- private void requireNonClosed() throws IOException {
- if (streamClosed) {
- throw new IOException("Stream closed.");
- }
- }
-
- /**
- * This method blocks until the stream is closed or enough bytes are
- * available, which can be read from the buffer.
- *
- * @param bytes the number of bytes waiting for.
- * @throws IOException If the thread is interrupted.
- * @return The available bytes.
- */
- private long tryWaitForEnoughBytes(final long bytes) throws IOException {
- // we can only wait for a positive number of bytes
- assert bytes > 0 : "Number of bytes are negative or zero : " + bytes;
-
- // if we haven't enough bytes, the loop starts and wait for enough bytes
- while (bytes > availableBytes) {
- try {
- // first of all, check for a closed stream
- if (streamClosed) {
- // is the stream closed, return only the current available bytes
- return availableBytes;
- }
- // wait for a next loop run and block until a modification is signalized
- signalModification.acquire();
- } catch (InterruptedException ex) {
- throw new IOException(ex);
- }
- }
- // return the available bytes (maybe higher as the required bytes)
- return availableBytes;
- }
-
- public class SBInputStream extends InputStream {
- @Override
- public int available() throws IOException {
- if (availableBytes > Integer.MAX_VALUE) {
- return Integer.MAX_VALUE;
- }
- return (int) availableBytes;
- }
-
- @Override
- public void close() throws IOException {
- closeAll();
- }
-
- @Override
- public int read() throws IOException {
- // we wait for enough bytes (one byte)
- if (tryWaitForEnoughBytes(1) < 1) {
- // try to wait, but not enough bytes available
- // return the end of stream is reached
- return -1;
- }
-
- // enough bytes are available, lock and modify the FIFO
- synchronized (bufferLock) {
- // get the first element from FIFO
- final byte[] first = buffer.getFirst();
- // get the first byte
- byte value = first[positionAtCurrentBufferEntry];
- // we have the first byte, now set the pointer to the next value
- ++positionAtCurrentBufferEntry;
- // if the pointer was pointed to the last element of the
- // byte array remove the first element from FIFO and reset the pointer
- if (positionAtCurrentBufferEntry >= first.length) {
- // reset the pointer
- positionAtCurrentBufferEntry = 0;
- // remove the first element from the buffer
- buffer.pollFirst();
- }
- availableBytes--;
- // returned as int in the range 0 to 255.
- return value & 0xff;
- }
- }
-
- // please do not override the method "int read(byte b[])"
- // the method calls internal "read(b, 0, b.length)"
- @Override
- public int read(final byte b[], final int off, final int len) throws IOException {
- if (!correctOffsetAndLengthToRead(b, off, len)) {
- return 0;
- }
-
- // try to read the first byte from FIFO
- // copied from super.read
- // === snip
- int c = read();
- if (c == -1) {
- return -1;
- }
- b[off] = (byte) c;
- // === snap
-
- // we have already copied one byte, initialize with 1
- int copiedBytes = 1;
-
- int missingBytes = len - copiedBytes;
- if (noMoreMissingBytes(missingBytes)) {
- return copiedBytes;
- }
-
- long maximumAvailableBytes = tryWaitForEnoughBytes(missingBytes);
-
- if (maximumAvailableBytes < 1) {
- // try to wait, but no more bytes available
- return copiedBytes;
- }
-
- // some or enough bytes are available, lock and modify the FIFO
- synchronized (bufferLock) {
- for (;;) {
-
- if (noMoreMissingBytes(missingBytes)) {
- return copiedBytes;
- }
-
- // get the first element from FIFO
- final byte[] first = buffer.getFirst();
- // get the maximum bytes which can be copied
- // from the first element
- final int maximumBytesToCopy
- = first.length - positionAtCurrentBufferEntry;
-
- // this element can be copied fully to the destination
- if (missingBytes >= maximumBytesToCopy) {
- // copy the complete byte[] to the destination
- System.arraycopy(first, positionAtCurrentBufferEntry, b,
- copiedBytes + off, maximumBytesToCopy);
- copiedBytes += maximumBytesToCopy;
- maximumAvailableBytes -= maximumBytesToCopy;
- availableBytes -= maximumBytesToCopy;
- missingBytes -= maximumBytesToCopy;
- // remove the first element from the buffer
- buffer.pollFirst();
- // reset the pointer
- positionAtCurrentBufferEntry = 0;
- } else {
- // copy only a part of byte[] to the destination
- System.arraycopy(first, positionAtCurrentBufferEntry, b,
- copiedBytes + off, missingBytes);
- // add the offset
- positionAtCurrentBufferEntry += missingBytes;
- copiedBytes += missingBytes;
- maximumAvailableBytes -= missingBytes;
- availableBytes -= missingBytes;
- // set missing bytes to zero
- // we reach the end of the current buffer (b)
- missingBytes = 0;
- }
- }
- }
- }
-
- /**
- * Ensure that no more bytes are missing.
- * @param missingBytes number of missing bytes.
- * @return <code>true</code> if no more bytes are missing, otherwise <code>false</code>.
- */
- private boolean noMoreMissingBytes(int missingBytes) {
- assert missingBytes >= 0 : "Copied more bytes as given";
-
- // check if we don't need to copy further bytes anymore
- return missingBytes == 0;
- }
-
- }
-
- public class SBOutputStream extends OutputStream {
- @Override
- public void close() throws IOException {
- closeAll();
- }
-
- @Override
- public void write(final int b) throws IOException {
- requireNonClosed();
- synchronized (bufferLock) {
- // add the byte to the buffer
- buffer.add(new byte[]{(byte) b});
- // increment the length
- ++availableBytes;
- // the count must be positive after any write operation
- assert availableBytes > 0 : "More memory used as a long can count";
- trim();
- }
- // always at least, signal bytes are written to the buffer
- signalModification();
- }
-
- // please do not override the method "void write(final byte[] b)"
- // the method calls internal "write(b, 0, b.length);"
- @Override
- public void write(final byte[] b, final int off, final int len)
- throws IOException {
- if (!correctOffsetAndLengthToWrite(b, off, len)) {
- return;
- }
- requireNonClosed();
- // To be thread safe cache the safeWrite value.
- boolean tmpSafeWrite = isSafeWrite();
-
- synchronized (bufferLock) {
- if (off == 0 && b.length == len) {
- // add the full byte[] to the buffer
- if (tmpSafeWrite && !ignoreSafeWrite) {
- buffer.add(b.clone());
- } else {
- buffer.add(b);
- }
- } else {
- byte[] target = new byte[len];
- System.arraycopy(b, off, target, 0, len);
- buffer.add(target);
- }
- // increment the length
- availableBytes += len;
- // the count must be positive after any write operation
- assert availableBytes > 0 : "More memory used as a long can count";
- trim();
- }
- // always at least, signal bytes are written to the buffer
- signalModification();
- }
- }
-
- public void close() throws IOException {
- closeAll();
- }
-
- /**
- * Invoke a close.
- */
- private void closeAll() {
- streamClosed = true;
- signalModification();
- }
-
- /**
- * Returns <code>true</code> if the stream is closed. Otherwise
- * <code>false</code>.
- *
- * @return <code>true</code> if the stream is closed. Otherwise
- * <code>false</code>.
- */
- public boolean isClosed() {
- return streamClosed;
- }
-
- /**
- * Returns an input stream for this buffer.
- *
- * <p>
- * Under abnormal conditions the underlying buffer may be closed. When the
- * buffer is closed the following applies to the returned input stream :-
- *
- * <ul>
- *
- * <li><p>
- * If there are no bytes buffered on the buffer, and the buffer has not been
- * closed using {@link #close close}, then
- * {@link java.io.InputStream#available available} will return
- * <code>0</code>.
- *
- * </ul>
- *
- * <p>
- * Closing the returned {@link java.io.InputStream InputStream} will close
- * the complete buffer.
- *
- * @return an input stream for reading bytes from this buffer.
- */
- public InputStream getInputStream() {
- return is;
- }
-
- /**
- * Returns an output stream for this buffer.
- *
- * <p>
- * Closing the returned {@link java.io.OutputStream OutputStream} will close
- * the associated buffer.
- *
- * @return an output stream for writing bytes to this buffer.
- */
- public OutputStream getOutputStream() {
- return os;
- }
-
- /**
- * Returns the number of elements in the buffer.
- *
- * @return the number of elements in the buffer.
- */
- public int getBufferSize() {
- synchronized (bufferLock) {
- return buffer.size();
- }
- }
-}
diff --git a/core/src/test/java/org/apache/airavata/mft/core/CircularStreamingBufferTest.java b/core/src/test/java/org/apache/airavata/mft/core/CircularStreamingBufferTest.java
new file mode 100644
index 0000000..69ea199
--- /dev/null
+++ b/core/src/test/java/org/apache/airavata/mft/core/CircularStreamingBufferTest.java
@@ -0,0 +1,265 @@
+package org.apache.airavata.mft.core;
+
+import java.io.*;
+import java.util.ArrayList;
+import java.util.List;
+
+public class CircularStreamingBufferTest {
+
+ public static void main(String args[]) throws Exception {
+ test3();
+ }
+
+ public static void test3() throws Exception {
+ final CircularStreamingBuffer sb = new CircularStreamingBuffer();
+ InputStream is = sb.getInputStream();
+ OutputStream os = sb.getOutputStream();
+ File file = new File("/Users/dimuthu/data.csv");
+ FileInputStream fis = new FileInputStream(file);
+
+ List<Byte> src = new ArrayList<>();
+ List<Byte> dst = new ArrayList<>();
+
+
+ Thread t1 = new Thread(new Runnable() {
+ @Override
+ public void run() {
+
+ try {
+ byte[] buf = new byte[1024];
+
+ int filesize = (int)file.length();
+ int bufSize;
+ while (true) {
+ if (buf.length < filesize) bufSize = buf.length;
+ else bufSize = (int) filesize;
+ bufSize = fis.read(buf, 0, bufSize);
+ if (bufSize < 0) {
+ // error
+ break;
+ }
+ System.out.println("Write " + bufSize);
+ os.write(buf, 0, bufSize);
+ os.flush();
+
+ for (int i = 0 ; i < bufSize; i++) {
+ src.add(buf[i]);
+ }
+
+ filesize -= bufSize;
+ if (filesize == 0L) break;
+ }
+ os.close();
+ System.out.println("Done writing");
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+ });
+
+
+ Thread t2 = new Thread(new Runnable() {
+ @Override
+ public void run() {
+ int res = 0;
+ byte[] buf = new byte[1024];
+
+ try {
+ int filesize = (int)file.length();
+
+ long totalWritten = 0;
+ while (true) {
+ int len = is.read(buf, 0, buf.length);
+ if (len == -1) {
+ break;
+ } else {
+ for (int i = 0; i < len; i++) {
+ dst.add(buf[i]);
+ }
+ totalWritten += len;
+ System.out.println("Read " + totalWritten);
+ if (totalWritten == filesize) {
+ break;
+ }
+ }
+ }
+ System.out.println("Done reading");
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+ });
+
+ t1.start();
+ t2.start();
+
+ t1.join();
+ t2.join();
+ System.out.println("Done");
+ System.out.println("Source length " + src.size());
+ System.out.println("Destination length " + dst.size());
+
+ boolean match = true;
+ for (int i = 0 ; i < src.size(); i++) {
+ match &= src.get(i).equals(dst.get(i));
+ if (!match) {
+ System.out.println("Failed at index " + i);
+ break;
+ }
+ }
+
+ System.out.println("Match : " + match);
+ }
+
+ public static void test2() throws Exception {
+ final CircularStreamingBuffer sb = new CircularStreamingBuffer();
+ InputStream is = sb.getInputStream();
+ OutputStream os = sb.getOutputStream();
+ File file = new File("/Users/dimuthu/data.csv");
+ FileInputStream fis = new FileInputStream(file);
+
+ List<Byte> src = new ArrayList<>();
+ List<Byte> dst = new ArrayList<>();
+
+
+ Thread t1 = new Thread(new Runnable() {
+ @Override
+ public void run() {
+
+ try {
+ byte[] buf = new byte[1024];
+
+ int filesize = (int)file.length();
+ int bufSize;
+ while (true) {
+ if (buf.length < filesize) bufSize = buf.length;
+ else bufSize = (int) filesize;
+ bufSize = fis.read(buf, 0, bufSize);
+ if (bufSize < 0) {
+ // error
+ break;
+ }
+ System.out.println("Write " + bufSize);
+ os.write(buf, 0, bufSize);
+ os.flush();
+
+ for (int i = 0 ; i < bufSize; i++) {
+ src.add(buf[i]);
+ }
+
+ filesize -= bufSize;
+ if (filesize == 0L) break;
+ }
+ os.close();
+ System.out.println("Done writing");
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+ });
+
+
+ Thread t2 = new Thread(new Runnable() {
+ @Override
+ public void run() {
+ int res = 0;
+ try {
+ while ((res = is.read()) != -1) {
+ System.out.println("Read " + res);
+ dst.add((byte) res);
+ }
+ System.out.println("Done reading");
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+ });
+
+ t1.start();
+ t2.start();
+
+ t1.join();
+ t2.join();
+ System.out.println("Done");
+ System.out.println("Source length " + src.size());
+ System.out.println("Destination length " + dst.size());
+
+ boolean match = true;
+ for (int i = 0 ; i < src.size(); i++) {
+ match &= src.get(i).equals(dst.get(i));
+ if (!match) {
+ System.out.println("Failed at index " + i);
+ break;
+ }
+ }
+
+ System.out.println("Match : " + match);
+ }
+
+ public static void test1() throws Exception {
+ final CircularStreamingBuffer sb = new CircularStreamingBuffer();
+ InputStream is = sb.getInputStream();
+ OutputStream os = sb.getOutputStream();
+ File file = new File("/Users/dimuthu/data.csv");
+ FileInputStream fis = new FileInputStream(file);
+
+ List<Byte> src = new ArrayList<>();
+ List<Byte> dst = new ArrayList<>();
+
+
+ Thread t1 = new Thread(new Runnable() {
+ @Override
+ public void run() {
+
+ try {
+ int read = 0;
+ while ((read = fis.read()) != -1) {
+ os.write(read);
+ src.add((byte) read);
+ }
+ os.close();
+ System.out.println("Done writing");
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+ });
+
+
+ Thread t2 = new Thread(new Runnable() {
+ @Override
+ public void run() {
+ int res = 0;
+ try {
+ while ((res = is.read()) != -1) {
+ System.out.println("Read " + res);
+ dst.add((byte) res);
+ }
+ System.out.println("Done reading");
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+ });
+
+ t1.start();
+ t2.start();
+
+ t1.join();
+ t2.join();
+ System.out.println("Done");
+ System.out.println("Source length " + src.size());
+ System.out.println("Destination length " + dst.size());
+
+ boolean match = true;
+ for (int i = 0 ; i < src.size(); i++) {
+ match &= src.get(i).equals(dst.get(i));
+ if (!match) {
+ System.out.println("Failed at index " + i);
+ break;
+ }
+ }
+
+ System.out.println("Match : " + match);
+ }
+}
diff --git a/transport/local-transport/pom.xml b/transport/local-transport/pom.xml
new file mode 100644
index 0000000..f067008
--- /dev/null
+++ b/transport/local-transport/pom.xml
@@ -0,0 +1,22 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <parent>
+ <artifactId>mft-transport</artifactId>
+ <groupId>org.apache.airavata</groupId>
+ <version>0.01-SNAPSHOT</version>
+ </parent>
+ <modelVersion>4.0.0</modelVersion>
+
+ <artifactId>mft-local-transport</artifactId>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.airavata</groupId>
+ <artifactId>mft-core</artifactId>
+ <version>0.01-SNAPSHOT</version>
+ </dependency>
+ </dependencies>
+
+</project>
\ No newline at end of file
diff --git a/transport/local-transport/src/main/java/org/apache/airavata/mft/transport/local/LocalMetadataCollector.java b/transport/local-transport/src/main/java/org/apache/airavata/mft/transport/local/LocalMetadataCollector.java
new file mode 100644
index 0000000..f72a6ea
--- /dev/null
+++ b/transport/local-transport/src/main/java/org/apache/airavata/mft/transport/local/LocalMetadataCollector.java
@@ -0,0 +1,20 @@
+package org.apache.airavata.mft.transport.local;
+
+import org.apache.airavata.mft.core.ResourceMetadata;
+import org.apache.airavata.mft.core.api.MetadataCollector;
+
+import java.io.File;
+
+public class LocalMetadataCollector implements MetadataCollector {
+ @Override
+ public ResourceMetadata getGetResourceMetadata(String resourceId, String credentialToken) throws Exception {
+
+ LocalResourceIdentifier resource = LocalTransportUtil.getLocalResourceIdentifier(resourceId);
+ File file = new File(resource.getPath());
+
+ ResourceMetadata metadata = new ResourceMetadata();
+ metadata.setResourceSize(file.length());
+ metadata.setUpdateTime(file.lastModified());
+ return metadata;
+ }
+}
diff --git a/transport/local-transport/src/main/java/org/apache/airavata/mft/transport/local/LocalReceiver.java b/transport/local-transport/src/main/java/org/apache/airavata/mft/transport/local/LocalReceiver.java
new file mode 100644
index 0000000..3bdee70
--- /dev/null
+++ b/transport/local-transport/src/main/java/org/apache/airavata/mft/transport/local/LocalReceiver.java
@@ -0,0 +1,59 @@
+package org.apache.airavata.mft.transport.local;
+
+import org.apache.airavata.mft.core.ConnectorContext;
+import org.apache.airavata.mft.core.api.Connector;
+
+import java.io.FileInputStream;
+import java.io.OutputStream;
+
+public class LocalReceiver implements Connector {
+
+ private LocalResourceIdentifier resource;
+ @Override
+ public void init(String resourceId, String credentialToken) throws Exception {
+ this.resource = LocalTransportUtil.getLocalResourceIdentifier(resourceId);
+ }
+
+ @Override
+ public void destroy() {
+
+ }
+
+ @Override
+ public void startStream(ConnectorContext context) throws Exception {
+ System.out.println("Starting local receive");
+ FileInputStream fin = new FileInputStream(this.resource.getPath());
+
+ long fileSize = context.getMetadata().getResourceSize();
+
+ OutputStream outputStream = context.getStreamBuffer().getOutputStream();
+
+ byte[] buf = new byte[1024];
+ while (true) {
+ int bufSize = 0;
+
+ if (buf.length < fileSize) {
+ bufSize = buf.length;
+ } else {
+ bufSize = (int) fileSize;
+ }
+ bufSize = fin.read(buf, 0, bufSize);
+
+ if (bufSize < 0) {
+ break;
+ }
+
+ outputStream.write(buf, 0, bufSize);
+ outputStream.flush();
+
+ fileSize -= bufSize;
+ if (fileSize == 0L)
+ break;
+ }
+
+ fin.close();
+ outputStream.close();
+ //context.getStreamBuffer().close();
+ System.out.println("Completed local receive");
+ }
+}
diff --git a/transport/local-transport/src/main/java/org/apache/airavata/mft/transport/local/LocalResourceIdentifier.java b/transport/local-transport/src/main/java/org/apache/airavata/mft/transport/local/LocalResourceIdentifier.java
new file mode 100644
index 0000000..5e1c528
--- /dev/null
+++ b/transport/local-transport/src/main/java/org/apache/airavata/mft/transport/local/LocalResourceIdentifier.java
@@ -0,0 +1,14 @@
+package org.apache.airavata.mft.transport.local;
+
+public class LocalResourceIdentifier {
+
+ private String path;
+
+ public String getPath() {
+ return path;
+ }
+
+ public void setPath(String path) {
+ this.path = path;
+ }
+}
diff --git a/transport/local-transport/src/main/java/org/apache/airavata/mft/transport/local/LocalSender.java b/transport/local-transport/src/main/java/org/apache/airavata/mft/transport/local/LocalSender.java
new file mode 100644
index 0000000..bd2b366
--- /dev/null
+++ b/transport/local-transport/src/main/java/org/apache/airavata/mft/transport/local/LocalSender.java
@@ -0,0 +1,54 @@
+package org.apache.airavata.mft.transport.local;
+
+import org.apache.airavata.mft.core.ConnectorContext;
+import org.apache.airavata.mft.core.api.Connector;
+
+import java.io.FileOutputStream;
+import java.io.InputStream;
+
+public class LocalSender implements Connector {
+
+ private LocalResourceIdentifier resource;
+ @Override
+ public void init(String resourceId, String credentialToken) throws Exception {
+ this.resource = LocalTransportUtil.getLocalResourceIdentifier(resourceId);
+ }
+
+ @Override
+ public void destroy() {
+
+ }
+
+ @Override
+ public void startStream(ConnectorContext context) throws Exception {
+ System.out.println("Starting local send");
+ FileOutputStream fos = new FileOutputStream(this.resource.getPath());
+ long fileSize = context.getMetadata().getResourceSize();
+
+ InputStream inputStream = context.getStreamBuffer().getInputStream();
+
+ byte[] buf = new byte[1];
+ while (true) {
+ int bufSize = 0;
+
+ if (buf.length < fileSize) {
+ bufSize = buf.length;
+ } else {
+ bufSize = (int) fileSize;
+ }
+ bufSize = inputStream.read(buf, 0, bufSize);
+
+ if (bufSize < 0) {
+ break;
+ }
+
+ fos.write(buf, 0, bufSize);
+ fos.flush();
+
+ fileSize -= bufSize;
+ if (fileSize == 0L)
+ break;
+ }
+ System.out.println("Completed local send");
+ }
+}
diff --git a/transport/local-transport/src/main/java/org/apache/airavata/mft/transport/local/LocalTransportUtil.java b/transport/local-transport/src/main/java/org/apache/airavata/mft/transport/local/LocalTransportUtil.java
new file mode 100644
index 0000000..4de7302
--- /dev/null
+++ b/transport/local-transport/src/main/java/org/apache/airavata/mft/transport/local/LocalTransportUtil.java
@@ -0,0 +1,18 @@
+package org.apache.airavata.mft.transport.local;
+
+public class LocalTransportUtil {
+
+ public static LocalResourceIdentifier getLocalResourceIdentifier(String id) {
+ LocalResourceIdentifier identifier = new LocalResourceIdentifier();
+
+ switch (id) {
+ case "1":
+ identifier.setPath("/Users/dimuthu/data.csv");
+ return identifier;
+ case "2":
+ identifier.setPath("/Users/dimuthu/new.txt");
+ return identifier;
+ }
+ return null;
+ }
+}
diff --git a/transport/pom.xml b/transport/pom.xml
index 13af130..64266c5 100755
--- a/transport/pom.xml
+++ b/transport/pom.xml
@@ -34,6 +34,7 @@
<packaging>pom</packaging>
<modules>
<module>scp-transport</module>
+ <module>local-transport</module>
</modules>
diff --git a/transport/scp-transport/src/main/java/org/apache/airavata/mft/transport/scp/SCPReceiver.java b/transport/scp-transport/src/main/java/org/apache/airavata/mft/transport/scp/SCPReceiver.java
index a639d36..7dd4339 100644
--- a/transport/scp-transport/src/main/java/org/apache/airavata/mft/transport/scp/SCPReceiver.java
+++ b/transport/scp-transport/src/main/java/org/apache/airavata/mft/transport/scp/SCPReceiver.java
@@ -2,8 +2,8 @@ package org.apache.airavata.mft.transport.scp;
import com.jcraft.jsch.ChannelExec;
import com.jcraft.jsch.Session;
+import org.apache.airavata.mft.core.CircularStreamingBuffer;
import org.apache.airavata.mft.core.ConnectorContext;
-import org.apache.airavata.mft.core.StreamBuffer;
import org.apache.airavata.mft.core.api.Connector;
import java.io.IOException;
@@ -35,7 +35,7 @@ public class SCPReceiver implements Connector {
transferRemoteToStream(session, sshResourceIdentifier.getRemotePath(), context.getStreamBuffer());
}
- private void transferRemoteToStream(Session session, String from, StreamBuffer streamBuffer) throws Exception {
+ private void transferRemoteToStream(Session session, String from, CircularStreamingBuffer streamBuffer) throws Exception {
OutputStream outputStream = streamBuffer.getOutputStream();
diff --git a/transport/scp-transport/src/main/java/org/apache/airavata/mft/transport/scp/SCPSender.java b/transport/scp-transport/src/main/java/org/apache/airavata/mft/transport/scp/SCPSender.java
index a1552ee..479bccf 100644
--- a/transport/scp-transport/src/main/java/org/apache/airavata/mft/transport/scp/SCPSender.java
+++ b/transport/scp-transport/src/main/java/org/apache/airavata/mft/transport/scp/SCPSender.java
@@ -3,8 +3,8 @@ package org.apache.airavata.mft.transport.scp;
import com.jcraft.jsch.ChannelExec;
import com.jcraft.jsch.JSchException;
import com.jcraft.jsch.Session;
+import org.apache.airavata.mft.core.CircularStreamingBuffer;
import org.apache.airavata.mft.core.ConnectorContext;
-import org.apache.airavata.mft.core.StreamBuffer;
import org.apache.airavata.mft.core.api.Connector;
import java.io.IOException;
@@ -34,7 +34,7 @@ public class SCPSender implements Connector {
copyLocalToRemote(this.session, sshResourceIdentifier.getRemotePath(), context.getStreamBuffer(), context.getMetadata().getResourceSize());
}
- private void copyLocalToRemote(Session session, String to, StreamBuffer streamBuffer, long fileSize) throws JSchException, IOException {
+ private void copyLocalToRemote(Session session, String to, CircularStreamingBuffer streamBuffer, long fileSize) throws JSchException, IOException {
System.out.println("Starting scp send");
InputStream inputStream = streamBuffer.getInputStream();
@@ -93,6 +93,7 @@ public class SCPSender implements Connector {
} else {
out.write(buf, 0, len); //out.flush();
totalWritten += len;
+ //System.out.println("Write " + totalWritten);
if (totalWritten == fileSize) {
break;
}