You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airavata.apache.org by di...@apache.org on 2019/12/11 07:34:28 UTC

[airavata-mft] branch master updated: Implementing a new streaming buffer and local transport

This is an automated email from the ASF dual-hosted git repository.

dimuthuupe pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/airavata-mft.git


The following commit(s) were added to refs/heads/master by this push:
     new ab9856b  Implementing a new streaming buffer and local transport
ab9856b is described below

commit ab9856b3102f3f4929856e78ef20acd646f4da1f
Author: Dimuthu Wannipurage <di...@gmail.com>
AuthorDate: Wed Dec 11 02:34:14 2019 -0500

    Implementing a new streaming buffer and local transport
---
 agent/pom.xml                                      |   5 +
 .../org/apache/airavata/mft/agent/MFTAgent.java    |  16 +-
 .../airavata/mft/agent/TransportMediator.java      |   5 +-
 .../airavata/mft/core/CircularStreamingBuffer.java | 116 ++++
 .../apache/airavata/mft/core/ConnectorContext.java |   6 +-
 .../org/apache/airavata/mft/core/StreamBuffer.java | 604 ---------------------
 .../mft/core/CircularStreamingBufferTest.java      | 265 +++++++++
 transport/local-transport/pom.xml                  |  22 +
 .../transport/local/LocalMetadataCollector.java    |  20 +
 .../mft/transport/local/LocalReceiver.java         |  59 ++
 .../transport/local/LocalResourceIdentifier.java   |  14 +
 .../airavata/mft/transport/local/LocalSender.java  |  54 ++
 .../mft/transport/local/LocalTransportUtil.java    |  18 +
 transport/pom.xml                                  |   1 +
 .../airavata/mft/transport/scp/SCPReceiver.java    |   4 +-
 .../airavata/mft/transport/scp/SCPSender.java      |   5 +-
 16 files changed, 599 insertions(+), 615 deletions(-)

diff --git a/agent/pom.xml b/agent/pom.xml
index 8559b82..a8846f5 100644
--- a/agent/pom.xml
+++ b/agent/pom.xml
@@ -38,5 +38,10 @@
             <artifactId>mft-scp-transport</artifactId>
             <version>0.01-SNAPSHOT</version>
         </dependency>
+        <dependency>
+            <groupId>org.apache.airavata</groupId>
+            <artifactId>mft-local-transport</artifactId>
+            <version>0.01-SNAPSHOT</version>
+        </dependency>
     </dependencies>
 </project>
\ No newline at end of file
diff --git a/agent/src/main/java/org/apache/airavata/mft/agent/MFTAgent.java b/agent/src/main/java/org/apache/airavata/mft/agent/MFTAgent.java
index de4fe72..1b050f0 100644
--- a/agent/src/main/java/org/apache/airavata/mft/agent/MFTAgent.java
+++ b/agent/src/main/java/org/apache/airavata/mft/agent/MFTAgent.java
@@ -3,6 +3,9 @@ package org.apache.airavata.mft.agent;
 import org.apache.airavata.mft.core.ResourceMetadata;
 import org.apache.airavata.mft.core.api.Connector;
 import org.apache.airavata.mft.core.api.MetadataCollector;
+import org.apache.airavata.mft.transport.local.LocalMetadataCollector;
+import org.apache.airavata.mft.transport.local.LocalReceiver;
+import org.apache.airavata.mft.transport.local.LocalSender;
 import org.apache.airavata.mft.transport.scp.SCPMetadataCollector;
 import org.apache.airavata.mft.transport.scp.SCPReceiver;
 import org.apache.airavata.mft.transport.scp.SCPSender;
@@ -26,7 +29,7 @@ public class MFTAgent {
 
                 MetadataCollector metadataCollector = resolveMetadataCollector(request.getSourceType());
                 ResourceMetadata metadata = metadataCollector.getGetResourceMetadata(request.getSourceId(), request.getSourceToken());
-
+                System.out.println("File size " + metadata.getResourceSize());
                 mediator.transfer(inConnector, outConnector, metadata);
 
             } catch (Exception e) {
@@ -39,7 +42,7 @@ public class MFTAgent {
         request.setSourceId("1");
         request.setSourceType("SCP");
         request.setDestinationId("2");
-        request.setDestinationType("SCP");
+        request.setDestinationType("LOCAL");
 
         MFTAgent agent = new MFTAgent();
         agent.requests.add(request);
@@ -57,6 +60,13 @@ public class MFTAgent {
                         return new SCPSender();
                 }
                 break;
+            case "LOCAL":
+                switch (direction) {
+                    case "IN":
+                        return new LocalReceiver();
+                    case "OUT":
+                        return new LocalSender();
+                }
         }
         return null;
     }
@@ -66,6 +76,8 @@ public class MFTAgent {
         switch (type) {
             case "SCP":
                 return new SCPMetadataCollector();
+            case "LOCAL":
+                return new LocalMetadataCollector();
         }
         return null;
     }
diff --git a/agent/src/main/java/org/apache/airavata/mft/agent/TransportMediator.java b/agent/src/main/java/org/apache/airavata/mft/agent/TransportMediator.java
index 319ee4e..f20570d 100644
--- a/agent/src/main/java/org/apache/airavata/mft/agent/TransportMediator.java
+++ b/agent/src/main/java/org/apache/airavata/mft/agent/TransportMediator.java
@@ -1,8 +1,8 @@
 package org.apache.airavata.mft.agent;
 
+import org.apache.airavata.mft.core.CircularStreamingBuffer;
 import org.apache.airavata.mft.core.ConnectorContext;
 import org.apache.airavata.mft.core.ResourceMetadata;
-import org.apache.airavata.mft.core.StreamBuffer;
 import org.apache.airavata.mft.core.TransferTask;
 import org.apache.airavata.mft.core.api.Connector;
 
@@ -20,7 +20,7 @@ public class TransportMediator {
 
     public void transfer(Connector inConnector, Connector outConnector, ResourceMetadata metadata) throws Exception {
 
-        StreamBuffer streamBuffer = new StreamBuffer();
+        CircularStreamingBuffer streamBuffer = new CircularStreamingBuffer();
         ConnectorContext context = new ConnectorContext();
         context.setMetadata(metadata);
         context.setStreamBuffer(streamBuffer);
@@ -46,6 +46,7 @@ public class TransportMediator {
                 System.out.println("One task failed with error: " + e.getMessage() );
                 e.printStackTrace();
                 for(Future<Integer> f : futureList){
+                    Thread.sleep(1000);
                     f.cancel(true);
                 }
                 futureList.clear();
diff --git a/core/src/main/java/org/apache/airavata/mft/core/CircularStreamingBuffer.java b/core/src/main/java/org/apache/airavata/mft/core/CircularStreamingBuffer.java
new file mode 100644
index 0000000..b18e0c3
--- /dev/null
+++ b/core/src/main/java/org/apache/airavata/mft/core/CircularStreamingBuffer.java
@@ -0,0 +1,116 @@
+package org.apache.airavata.mft.core;
+
+import java.io.*;
+import java.util.concurrent.ArrayBlockingQueue;
+import java.util.concurrent.Semaphore;
+
+public class CircularStreamingBuffer {
+
+    private int bufferSize = 1024;
+    private ArrayBlockingQueue<Byte> buffer = new ArrayBlockingQueue<>(bufferSize);
+
+    private boolean osClosed = false;
+    private Semaphore readSem = new Semaphore(0);
+
+    private OutputStream outputStream = new CSBOutputStream();
+    private InputStream inputStream = new CSBInputStream();
+
+    public OutputStream getOutputStream() {
+        return outputStream;
+    }
+
+    public InputStream getInputStream() {
+        return inputStream;
+    }
+
+    private void updateRead() {
+        if (readSem.availablePermits() == 0) {
+            readSem.release();
+        }
+    }
+
+    public class CSBOutputStream extends OutputStream {
+        @Override
+        public void write(byte[] b) throws IOException {
+            write(b, 0, b.length);
+        }
+
+        @Override
+        public void write(byte[] b, int off, int len) throws IOException {
+            for (int i = off; i < len; i ++) {
+                try {
+                    buffer.put(b[i]);
+                    updateRead();
+                } catch (InterruptedException e) {
+                    throw new IOException(e);
+                }
+            }
+        }
+
+        @Override
+        public void flush() throws IOException {
+        }
+
+        @Override
+        public void close() throws IOException {
+            osClosed = true;
+            updateRead();
+        }
+
+        @Override
+        public void write(int b) throws IOException {
+            try {
+                buffer.put((byte)b);
+                //System.out.println("Writing");
+                updateRead();
+            } catch (InterruptedException e) {
+                throw new IOException(e);
+            }
+        }
+    }
+
+    public class CSBInputStream extends InputStream {
+
+        @Override
+        public int read() throws IOException {
+            Byte res = buffer.poll();
+            if (res == null) {
+                //System.out.println("Received null in is.read()");
+                if (osClosed) return -1;
+                try {
+                    readSem.acquire();
+                    return read();
+                } catch (InterruptedException e) {
+                    throw new IOException(e);
+                }
+            } else {
+                return res & 0xff;
+            }
+        }
+
+        @Override
+        public int read(byte[] b, int off, int len) throws IOException {
+            for (int i = off; i < len; i++) {
+                int res = read();
+                if (res == -1) {
+                    //System.out.println("Received -1 in is.read(byte[], off, len)");
+                    if (i == off) {
+                        //System.out.println("Return -1");
+                        return -1;
+                    } else {
+                        //System.out.println("Return " + (i - off));
+                        return i - off;
+                    }
+                } else {
+                    b[i] = (byte)res;
+                }
+            }
+            return len - off;
+        }
+
+        @Override
+        public int read(byte[] b) throws IOException {
+            return read(b, 0, b.length);
+        }
+    }
+}
\ No newline at end of file
diff --git a/core/src/main/java/org/apache/airavata/mft/core/ConnectorContext.java b/core/src/main/java/org/apache/airavata/mft/core/ConnectorContext.java
index 24485c5..2592108 100644
--- a/core/src/main/java/org/apache/airavata/mft/core/ConnectorContext.java
+++ b/core/src/main/java/org/apache/airavata/mft/core/ConnectorContext.java
@@ -2,14 +2,14 @@ package org.apache.airavata.mft.core;
 
 public class ConnectorContext {
 
-    private StreamBuffer streamBuffer;
+    private CircularStreamingBuffer streamBuffer;
     private ResourceMetadata metadata;
 
-    public StreamBuffer getStreamBuffer() {
+    public CircularStreamingBuffer getStreamBuffer() {
         return streamBuffer;
     }
 
-    public void setStreamBuffer(StreamBuffer streamBuffer) {
+    public void setStreamBuffer(CircularStreamingBuffer streamBuffer) {
         this.streamBuffer = streamBuffer;
     }
 
diff --git a/core/src/main/java/org/apache/airavata/mft/core/StreamBuffer.java b/core/src/main/java/org/apache/airavata/mft/core/StreamBuffer.java
deleted file mode 100644
index 715d3ef..0000000
--- a/core/src/main/java/org/apache/airavata/mft/core/StreamBuffer.java
+++ /dev/null
@@ -1,604 +0,0 @@
-// @formatter:off
-/**
- * Copyright 2014 Bernard Ladenthin bernard.ladenthin@gmail.com
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-// @formatter:on
-package org.apache.airavata.mft.core;
-
-import java.io.Closeable;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.util.Deque;
-import java.util.LinkedList;
-import java.util.concurrent.Semaphore;
-
-// Copied from https://github.com/bernardladenthin/streambuffer
-/**
- * A stream buffer is a class to buffer data that has been written to an
- * {@link OutputStream} and provide the data in an {@link InputStream}. There is
- * no need to close the stream before read. The read works during a write
- * process. It is possible to call concurrent a method in the
- * {@link OutputStream} and {@link InputStream}. Read/Write at the same time.
- *
- * @author Bernard Ladenthin bernard.ladenthin@gmail.com
- */
-public class StreamBuffer implements Closeable {
-
-    /**
-     * An object to get an unique access to the {@link #buffer}. It is needed to
-     * get an exclusive access for read and write operations.
-     */
-    private final Object bufferLock = new Object();
-
-    /**
-     * The buffer which contains the raw data.
-     */
-    private final Deque<byte[]> buffer = new LinkedList<>();
-
-    /**
-     * A {@link Semaphore} to signal that data are added to the {@link #buffer}.
-     * This signal is also used to announce that the stream was closed.
-     * This {@link Semaphore} is used only for internal communication.
-     */
-    private final Semaphore signalModification = new Semaphore(0);
-
-    /**
-     * A {@link Semaphore} to signal that data are added to the {@link #buffer}.
-     * This signal is also used to announce that the stream was closed.
-     * This {@link Semaphore} is used only for external communication.
-     */
-    private final Semaphore signalModificationExternal = new Semaphore(0);
-
-    /**
-     * A variable for the current position of the current element in the
-     * {@link #buffer}.
-     */
-    private volatile int positionAtCurrentBufferEntry = 0;
-
-    /**
-     * The sum of available bytes.
-     */
-    private volatile long availableBytes = 0;
-
-    /**
-     * A flag to indicate the stream was closed.
-     */
-    private volatile boolean streamClosed = false;
-
-    /**
-     * A flag to enable a safe write. If safe write is enabled, modifiable byte
-     * arrays are cloned before they are written (put) into the FIFO. Benefit:
-     * It cost more performance to clone a byte array, but the content is
-     * immutable. This may be revoked by {@link #ignoreSafeWrite}, for example a
-     * byte array was created internally.
-     */
-    private volatile boolean safeWrite = false;
-
-    /**
-     * A flag to disable the {@link StreamBuffer#safeWrite}. I. e. to write a byte array from
-     * the trim method.
-     */
-    private boolean ignoreSafeWrite = false;
-
-    /**
-     * The maximum buffer elements. A number of maximum elements to invoke the
-     * trim method. If the buffer contains more elements as the maximum, the
-     * buffer will be completely reduced to one element. This option can help to
-     * improve the performance for a read of huge amount data. It can also help
-     * to shrink the size of the memory used. Use a value lower or equals
-     * <code>0</code> to disable the trim option. Default is a maximum of
-     * <code>100</code> elements.
-     */
-    private volatile int maxBufferElements = 100;
-
-    private final SBInputStream is = new SBInputStream();
-    private final SBOutputStream os = new SBOutputStream();
-
-    /**
-     * Construct a new {@link StreamBuffer}.
-     */
-    public StreamBuffer() {
-    }
-
-    /**
-     * Returns the flag which indicates whether write operations are executed
-     * safe or unsafe.
-     *
-     * @return <code>true</code> if the safe write option is enabled, otherwise
-     * <code>false</code>.
-     */
-    public boolean isSafeWrite() {
-        return safeWrite;
-    }
-
-    /**
-     * Set a secure or unsercure write operation.
-     *
-     * @param safeWrite A flag to enable a safe write. If safe write is enabled,
-     * modifiable byte arrays are cloned before they are written. Benefit: It
-     * cost more performance to clone a byte array, but it is hardened to
-     * external changes. Use <code>true</code> to force a clone. Otherwise use
-     * <code>false</code> (default).
-     */
-    public void setSafeWrite(boolean safeWrite) {
-        this.safeWrite = safeWrite;
-    }
-
-    /**
-     * Returns the number of maximum elements which are held in maximum memory.
-     *
-     * @return the number of elements.
-     */
-    public int getMaxBufferElements() {
-        return maxBufferElements;
-    }
-
-    /**
-     * Set maximum elements for the buffer. Change the value doesn't invoke a
-     * trim call if the buffer contains more elements. Only write operations
-     * force a trim call.
-     *
-     * @param maxBufferElements number of maximum elements.
-     */
-    public void setMaxBufferElements(int maxBufferElements) {
-        this.maxBufferElements = maxBufferElements;
-    }
-
-    /**
-     * Security check mostly copied from {@link InputStream#read(byte[], int, int)}.
-     * Ensures the parameter are valid.
-     * @param b the byte array to copy from
-     * @param off the offset to read from the array
-     * @param len the len of the bytes to read from the array
-     * @return <code>true</code> if there are bytes to read, otherwise <code>false</code>
-     * @throws NullPointerException if the array is null
-     * @throws IndexOutOfBoundsException if the index is not correct
-     */
-    public static boolean correctOffsetAndLengthToRead(byte[] b, int off, int len) {
-        if (b == null) {
-            throw new NullPointerException();
-        } else if (off < 0 || len < 0 || len > b.length - off) {
-            throw new IndexOutOfBoundsException();
-        } else if (len == 0) {
-            return false;
-        }
-        return true;
-    }
-
-    /**
-     * Security check mostly copied from {@link OutputStream#write(byte[], int, int)}.
-     * Ensures the parameter are valid.
-     * @param b the byte array to write to the array
-     * @param off the offset to write to the array
-     * @param len the len of the bytes to write to the array
-     * @return <code>true</code> if there are bytes to write, otherwise <code>false</code>
-     * @throws NullPointerException if the array is null
-     * @throws IndexOutOfBoundsException if the index is not correct
-     */
-    public static boolean correctOffsetAndLengthToWrite(byte[] b, int off, int len) {
-        if (b == null) {
-            throw new NullPointerException();
-        } else if ((off < 0) || (off > b.length) || (len < 0)
-                || ((off + len) > b.length) || ((off + len) < 0)) {
-            throw new IndexOutOfBoundsException();
-        } else if (len == 0) {
-            return false;
-        }
-        return true;
-    }
-
-    /**
-     * Call this method always after all changes are synchronized. This method
-     * signals a modification. It cloud be a write on the stream or a close.
-     */
-    private void signalModification() {
-        // hold up the permits to a maximum of one
-        if (signalModification.availablePermits() == 0) {
-            signalModification.release();
-        }
-        // same twice for external communication
-        if (signalModificationExternal.availablePermits() == 0) {
-            signalModificationExternal.release();
-        }
-    }
-
-    /**
-     * This method is blocking until data is available on the
-     * {@link InputStream} or the stream was closed. This method could be called
-     * from one thread only. This method could not be used to notify two threads.
-     * @throws java.lang.InterruptedException if the current thread is interrupted
-     */
-    public void blockDataAvailable() throws InterruptedException {
-        if (isClosed()) {
-            return;
-        }
-
-        if (availableBytes < 1) {
-            signalModificationExternal.acquire();
-        }
-    }
-
-    /**
-     * This method trims the buffer. This method can be invoked after every
-     * write operation. The method checks itself if the buffer should be trimmed
-     * or not.
-     */
-    private void trim() throws IOException {
-        if (isTrimShouldBeExecuted()) {
-
-            /**
-             * Need to store more bufs, may it is not possible to read out all
-             * data at once. The available method only returns an int value
-             * instead a long value. Store all read parts of the full buffer in
-             * a deque.
-             */
-            final Deque<byte[]> tmpBuffer = new LinkedList<>();
-
-            int available;
-            // empty the current buffer, read out all bytes
-            while ((available = is.available()) > 0) {
-                final byte[] buf = new byte[available];
-                // read out of the buffer
-                // and store the result to the tmpBuffer
-                int read = is.read(buf);
-                // should never happen
-                assert read == available : "Read not enough bytes from buffer.";
-                tmpBuffer.add(buf);
-            }
-            /**
-             * Write all previously read parts back to the buffer. The buffer is
-             * clean and contains no elements because all parts are read out.
-             */
-            try {
-                /**
-                 * The reference to the buffer deque is not reachable out of the
-                 * trim method. It is possible to ignore the safeWrite flag to
-                 * prevent not necessary clone operations.
-                 */
-                ignoreSafeWrite = true;
-                while (!tmpBuffer.isEmpty()) {
-                    // pollFirst returns always a non null value, tmpBuffer is only filled with non null values
-                    os.write(tmpBuffer.pollFirst());
-                }
-            } finally {
-                ignoreSafeWrite = false;
-            }
-        }
-    }
-
-    /**
-     * Checks if a trim should be performed.
-     * @return <code>true</code> if a trim should be performed, otherwise <code>false</code>.
-     */
-    private boolean isTrimShouldBeExecuted() {
-        /**
-         * To be thread safe, cache the maxBufferElements value. May the method
-         * {@link #setMaxBufferElements(int)} was invoked from outside by another thread.
-         */
-        final int maxBufferElements = getMaxBufferElements();
-        return (maxBufferElements > 0) && (buffer.size() >= 2) && (buffer.size() > maxBufferElements);
-    }
-
-    /**
-     * This method mustn't be called in a synchronized context, the variable is
-     * volatile.
-     */
-    private void requireNonClosed() throws IOException {
-        if (streamClosed) {
-            throw new IOException("Stream closed.");
-        }
-    }
-
-    /**
-     * This method blocks until the stream is closed or enough bytes are
-     * available, which can be read from the buffer.
-     *
-     * @param bytes the number of bytes waiting for.
-     * @throws IOException If the thread is interrupted.
-     * @return The available bytes.
-     */
-    private long tryWaitForEnoughBytes(final long bytes) throws IOException {
-        // we can only wait for a positive number of bytes
-        assert bytes > 0 : "Number of bytes are negative or zero : " + bytes;
-
-        // if we haven't enough bytes, the loop starts and wait for enough bytes
-        while (bytes > availableBytes) {
-            try {
-                // first of all, check for a closed stream
-                if (streamClosed) {
-                    // is the stream closed, return only the current available bytes
-                    return availableBytes;
-                }
-                // wait for a next loop run and block until a modification is signalized
-                signalModification.acquire();
-            } catch (InterruptedException ex) {
-                throw new IOException(ex);
-            }
-        }
-        // return the available bytes (maybe higher as the required bytes)
-        return availableBytes;
-    }
-
-    public class SBInputStream extends InputStream {
-        @Override
-        public int available() throws IOException {
-            if (availableBytes > Integer.MAX_VALUE) {
-                return Integer.MAX_VALUE;
-            }
-            return (int) availableBytes;
-        }
-
-        @Override
-        public void close() throws IOException {
-            closeAll();
-        }
-
-        @Override
-        public int read() throws IOException {
-            // we wait for enough bytes (one byte)
-            if (tryWaitForEnoughBytes(1) < 1) {
-                // try to wait, but not enough bytes available
-                // return the end of stream is reached
-                return -1;
-            }
-
-            // enough bytes are available, lock and modify the FIFO
-            synchronized (bufferLock) {
-                // get the first element from FIFO
-                final byte[] first = buffer.getFirst();
-                // get the first byte
-                byte value = first[positionAtCurrentBufferEntry];
-                // we have the first byte, now set the pointer to the next value
-                ++positionAtCurrentBufferEntry;
-                // if the pointer was pointed to the last element of the
-                // byte array remove the first element from FIFO and reset the pointer
-                if (positionAtCurrentBufferEntry >= first.length) {
-                    // reset the pointer
-                    positionAtCurrentBufferEntry = 0;
-                    // remove the first element from the buffer
-                    buffer.pollFirst();
-                }
-                availableBytes--;
-                // returned as int in the range 0 to 255.
-                return value & 0xff;
-            }
-        }
-
-        // please do not override the method "int read(byte b[])"
-        // the method calls internal "read(b, 0, b.length)"
-        @Override
-        public int read(final byte b[], final int off, final int len) throws IOException {
-            if (!correctOffsetAndLengthToRead(b, off, len)) {
-                return 0;
-            }
-
-            // try to read the first byte from FIFO
-            // copied from super.read
-            // === snip
-            int c = read();
-            if (c == -1) {
-                return -1;
-            }
-            b[off] = (byte) c;
-            // === snap
-
-            // we have already copied one byte, initialize with 1
-            int copiedBytes = 1;
-
-            int missingBytes = len - copiedBytes;
-            if (noMoreMissingBytes(missingBytes)) {
-                return copiedBytes;
-            }
-
-            long maximumAvailableBytes = tryWaitForEnoughBytes(missingBytes);
-
-            if (maximumAvailableBytes < 1) {
-                // try to wait, but no more bytes available
-                return copiedBytes;
-            }
-
-            // some or enough bytes are available, lock and modify the FIFO
-            synchronized (bufferLock) {
-                for (;;) {
-
-                    if (noMoreMissingBytes(missingBytes)) {
-                        return copiedBytes;
-                    }
-
-                    // get the first element from FIFO
-                    final byte[] first = buffer.getFirst();
-                    // get the maximum bytes which can be copied
-                    // from the first element
-                    final int maximumBytesToCopy
-                            = first.length - positionAtCurrentBufferEntry;
-
-                    // this element can be copied fully to the destination
-                    if (missingBytes >= maximumBytesToCopy) {
-                        // copy the complete byte[] to the destination
-                        System.arraycopy(first, positionAtCurrentBufferEntry, b,
-                                copiedBytes + off, maximumBytesToCopy);
-                        copiedBytes += maximumBytesToCopy;
-                        maximumAvailableBytes -= maximumBytesToCopy;
-                        availableBytes -= maximumBytesToCopy;
-                        missingBytes -= maximumBytesToCopy;
-                        // remove the first element from the buffer
-                        buffer.pollFirst();
-                        // reset the pointer
-                        positionAtCurrentBufferEntry = 0;
-                    } else {
-                        // copy only a part of byte[] to the destination
-                        System.arraycopy(first, positionAtCurrentBufferEntry, b,
-                                copiedBytes + off, missingBytes);
-                        // add the offset
-                        positionAtCurrentBufferEntry += missingBytes;
-                        copiedBytes += missingBytes;
-                        maximumAvailableBytes -= missingBytes;
-                        availableBytes -= missingBytes;
-                        // set missing bytes to zero
-                        // we reach the end of the current buffer (b)
-                        missingBytes = 0;
-                    }
-                }
-            }
-        }
-
-        /**
-         * Ensure that no more bytes are missing.
-         * @param missingBytes number of missing bytes.
-         * @return <code>true</code> if no more bytes are missing, otherwise <code>false</code>.
-         */
-        private boolean noMoreMissingBytes(int missingBytes) {
-            assert missingBytes >= 0 : "Copied more bytes as given";
-
-            // check if we don't need to copy further bytes anymore
-            return missingBytes == 0;
-        }
-
-    }
-
-    public class SBOutputStream extends OutputStream {
-        @Override
-        public void close() throws IOException {
-            closeAll();
-        }
-
-        @Override
-        public void write(final int b) throws IOException {
-            requireNonClosed();
-            synchronized (bufferLock) {
-                // add the byte to the buffer
-                buffer.add(new byte[]{(byte) b});
-                // increment the length
-                ++availableBytes;
-                // the count must be positive after any write operation
-                assert availableBytes > 0 : "More memory used as a long can count";
-                trim();
-            }
-            // always at least, signal bytes are written to the buffer
-            signalModification();
-        }
-
-        // please do not override the method "void write(final byte[] b)"
-        // the method calls internal "write(b, 0, b.length);"
-        @Override
-        public void write(final byte[] b, final int off, final int len)
-                throws IOException {
-            if (!correctOffsetAndLengthToWrite(b, off, len)) {
-                return;
-            }
-            requireNonClosed();
-            // To be thread safe cache the safeWrite value.
-            boolean tmpSafeWrite = isSafeWrite();
-
-            synchronized (bufferLock) {
-                if (off == 0 && b.length == len) {
-                    // add the full byte[] to the buffer
-                    if (tmpSafeWrite && !ignoreSafeWrite) {
-                        buffer.add(b.clone());
-                    } else {
-                        buffer.add(b);
-                    }
-                } else {
-                    byte[] target = new byte[len];
-                    System.arraycopy(b, off, target, 0, len);
-                    buffer.add(target);
-                }
-                // increment the length
-                availableBytes += len;
-                // the count must be positive after any write operation
-                assert availableBytes > 0 : "More memory used as a long can count";
-                trim();
-            }
-            // always at least, signal bytes are written to the buffer
-            signalModification();
-        }
-    }
-
-    public void close() throws IOException {
-        closeAll();
-    }
-
-    /**
-     * Invoke a close.
-     */
-    private void closeAll() {
-        streamClosed = true;
-        signalModification();
-    }
-
-    /**
-     * Returns <code>true</code> if the stream is closed. Otherwise
-     * <code>false</code>.
-     *
-     * @return <code>true</code> if the stream is closed. Otherwise
-     * <code>false</code>.
-     */
-    public boolean isClosed() {
-        return streamClosed;
-    }
-
-    /**
-     * Returns an input stream for this buffer.
-     *
-     * <p>
-     * Under abnormal conditions the underlying buffer may be closed. When the
-     * buffer is closed the following applies to the returned input stream :-
-     *
-     * <ul>
-     *
-     * <li><p>
-     * If there are no bytes buffered on the buffer, and the buffer has not been
-     * closed using {@link #close close}, then
-     * {@link java.io.InputStream#available available} will return
-     * <code>0</code>.
-     *
-     * </ul>
-     *
-     * <p>
-     * Closing the returned {@link java.io.InputStream InputStream} will close
-     * the complete buffer.
-     *
-     * @return an input stream for reading bytes from this buffer.
-     */
-    public InputStream getInputStream() {
-        return is;
-    }
-
-    /**
-     * Returns an output stream for this buffer.
-     *
-     * <p>
-     * Closing the returned {@link java.io.OutputStream OutputStream} will close
-     * the associated buffer.
-     *
-     * @return an output stream for writing bytes to this buffer.
-     */
-    public OutputStream getOutputStream() {
-        return os;
-    }
-
-    /**
-     * Returns the number of elements in the buffer.
-     *
-     * @return the number of elements in the buffer.
-     */
-    public int getBufferSize() {
-        synchronized (bufferLock) {
-            return buffer.size();
-        }
-    }
-}
diff --git a/core/src/test/java/org/apache/airavata/mft/core/CircularStreamingBufferTest.java b/core/src/test/java/org/apache/airavata/mft/core/CircularStreamingBufferTest.java
new file mode 100644
index 0000000..69ea199
--- /dev/null
+++ b/core/src/test/java/org/apache/airavata/mft/core/CircularStreamingBufferTest.java
@@ -0,0 +1,265 @@
+package org.apache.airavata.mft.core;
+
+import java.io.*;
+import java.util.ArrayList;
+import java.util.List;
+
+public class CircularStreamingBufferTest {
+
+    public static void main(String args[]) throws Exception {
+        test3();
+    }
+
+    public static void test3() throws Exception {
+        final CircularStreamingBuffer sb = new CircularStreamingBuffer();
+        InputStream is = sb.getInputStream();
+        OutputStream os = sb.getOutputStream();
+        File file = new File("/Users/dimuthu/data.csv");
+        FileInputStream fis = new FileInputStream(file);
+
+        List<Byte> src = new ArrayList<>();
+        List<Byte> dst = new ArrayList<>();
+
+
+        Thread t1 = new Thread(new Runnable() {
+            @Override
+            public void run() {
+
+                try {
+                    byte[] buf = new byte[1024];
+
+                    int filesize = (int)file.length();
+                    int bufSize;
+                    while (true) {
+                        if (buf.length < filesize) bufSize = buf.length;
+                        else bufSize = (int) filesize;
+                        bufSize = fis.read(buf, 0, bufSize);
+                        if (bufSize < 0) {
+                            // error
+                            break;
+                        }
+                        System.out.println("Write " + bufSize);
+                        os.write(buf, 0, bufSize);
+                        os.flush();
+
+                        for (int i = 0 ; i < bufSize; i++) {
+                            src.add(buf[i]);
+                        }
+
+                        filesize -= bufSize;
+                        if (filesize == 0L) break;
+                    }
+                    os.close();
+                    System.out.println("Done writing");
+                } catch (Exception e) {
+                    e.printStackTrace();
+                }
+            }
+        });
+
+
+        Thread t2 = new Thread(new Runnable() {
+            @Override
+            public void run() {
+                int res = 0;
+                byte[] buf = new byte[1024];
+
+                try {
+                    int filesize = (int)file.length();
+
+                    long totalWritten = 0;
+                    while (true) {
+                        int len = is.read(buf, 0, buf.length);
+                        if (len == -1) {
+                            break;
+                        } else {
+                            for (int i = 0; i < len; i++) {
+                                dst.add(buf[i]);
+                            }
+                            totalWritten += len;
+                            System.out.println("Read " + totalWritten);
+                            if (totalWritten == filesize) {
+                                break;
+                            }
+                        }
+                    }
+                    System.out.println("Done reading");
+                } catch (Exception e) {
+                    e.printStackTrace();
+                }
+            }
+        });
+
+        t1.start();
+        t2.start();
+
+        t1.join();
+        t2.join();
+        System.out.println("Done");
+        System.out.println("Source length " + src.size());
+        System.out.println("Destination length " + dst.size());
+
+        boolean match = true;
+        for (int i = 0 ; i < src.size(); i++) {
+            match &= src.get(i).equals(dst.get(i));
+            if (!match) {
+                System.out.println("Failed at index " + i);
+                break;
+            }
+        }
+
+        System.out.println("Match : " + match);
+    }
+
+    public static void test2() throws Exception {
+        final CircularStreamingBuffer sb = new CircularStreamingBuffer();
+        InputStream is = sb.getInputStream();
+        OutputStream os = sb.getOutputStream();
+        File file = new File("/Users/dimuthu/data.csv");
+        FileInputStream fis = new FileInputStream(file);
+
+        List<Byte> src = new ArrayList<>();
+        List<Byte> dst = new ArrayList<>();
+
+
+        Thread t1 = new Thread(new Runnable() {
+            @Override
+            public void run() {
+
+                try {
+                    byte[] buf = new byte[1024];
+
+                    int filesize = (int)file.length();
+                    int bufSize;
+                    while (true) {
+                        if (buf.length < filesize) bufSize = buf.length;
+                        else bufSize = (int) filesize;
+                        bufSize = fis.read(buf, 0, bufSize);
+                        if (bufSize < 0) {
+                            // error
+                            break;
+                        }
+                        System.out.println("Write " + bufSize);
+                        os.write(buf, 0, bufSize);
+                        os.flush();
+
+                        for (int i = 0 ; i < bufSize; i++) {
+                            src.add(buf[i]);
+                        }
+
+                        filesize -= bufSize;
+                        if (filesize == 0L) break;
+                    }
+                    os.close();
+                    System.out.println("Done writing");
+                } catch (Exception e) {
+                    e.printStackTrace();
+                }
+            }
+        });
+
+
+        Thread t2 = new Thread(new Runnable() {
+            @Override
+            public void run() {
+                int res = 0;
+                try {
+                    while ((res = is.read()) != -1) {
+                        System.out.println("Read " + res);
+                        dst.add((byte) res);
+                    }
+                    System.out.println("Done reading");
+                } catch (Exception e) {
+                    e.printStackTrace();
+                }
+            }
+        });
+
+        t1.start();
+        t2.start();
+
+        t1.join();
+        t2.join();
+        System.out.println("Done");
+        System.out.println("Source length " + src.size());
+        System.out.println("Destination length " + dst.size());
+
+        boolean match = true;
+        for (int i = 0 ; i < src.size(); i++) {
+            match &= src.get(i).equals(dst.get(i));
+            if (!match) {
+                System.out.println("Failed at index " + i);
+                break;
+            }
+        }
+
+        System.out.println("Match : " + match);
+    }
+
+    public static void test1() throws Exception {
+        final CircularStreamingBuffer sb = new CircularStreamingBuffer();
+        InputStream is = sb.getInputStream();
+        OutputStream os = sb.getOutputStream();
+        File file = new File("/Users/dimuthu/data.csv");
+        FileInputStream fis = new FileInputStream(file);
+
+        List<Byte> src = new ArrayList<>();
+        List<Byte> dst = new ArrayList<>();
+
+
+        Thread t1 = new Thread(new Runnable() {
+            @Override
+            public void run() {
+
+                try {
+                    int read = 0;
+                    while ((read = fis.read()) != -1) {
+                        os.write(read);
+                        src.add((byte) read);
+                    }
+                    os.close();
+                    System.out.println("Done writing");
+                } catch (Exception e) {
+                    e.printStackTrace();
+                }
+            }
+        });
+
+
+        Thread t2 = new Thread(new Runnable() {
+            @Override
+            public void run() {
+                int res = 0;
+                try {
+                    while ((res = is.read()) != -1) {
+                        System.out.println("Read " + res);
+                        dst.add((byte) res);
+                    }
+                    System.out.println("Done reading");
+                } catch (Exception e) {
+                    e.printStackTrace();
+                }
+            }
+        });
+
+        t1.start();
+        t2.start();
+
+        t1.join();
+        t2.join();
+        System.out.println("Done");
+        System.out.println("Source length " + src.size());
+        System.out.println("Destination length " + dst.size());
+
+        boolean match = true;
+        for (int i = 0 ; i < src.size(); i++) {
+            match &= src.get(i).equals(dst.get(i));
+            if (!match) {
+                System.out.println("Failed at index " + i);
+                break;
+            }
+        }
+
+        System.out.println("Match : " + match);
+    }
+}
diff --git a/transport/local-transport/pom.xml b/transport/local-transport/pom.xml
new file mode 100644
index 0000000..f067008
--- /dev/null
+++ b/transport/local-transport/pom.xml
@@ -0,0 +1,22 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <parent>
+        <artifactId>mft-transport</artifactId>
+        <groupId>org.apache.airavata</groupId>
+        <version>0.01-SNAPSHOT</version>
+    </parent>
+    <modelVersion>4.0.0</modelVersion>
+
+    <artifactId>mft-local-transport</artifactId>
+
+    <dependencies>
+        <dependency>
+            <groupId>org.apache.airavata</groupId>
+            <artifactId>mft-core</artifactId>
+            <version>0.01-SNAPSHOT</version>
+        </dependency>
+    </dependencies>
+
+</project>
\ No newline at end of file
diff --git a/transport/local-transport/src/main/java/org/apache/airavata/mft/transport/local/LocalMetadataCollector.java b/transport/local-transport/src/main/java/org/apache/airavata/mft/transport/local/LocalMetadataCollector.java
new file mode 100644
index 0000000..f72a6ea
--- /dev/null
+++ b/transport/local-transport/src/main/java/org/apache/airavata/mft/transport/local/LocalMetadataCollector.java
@@ -0,0 +1,20 @@
+package org.apache.airavata.mft.transport.local;
+
+import org.apache.airavata.mft.core.ResourceMetadata;
+import org.apache.airavata.mft.core.api.MetadataCollector;
+
+import java.io.File;
+
+public class LocalMetadataCollector implements MetadataCollector {
+    @Override
+    public ResourceMetadata getGetResourceMetadata(String resourceId, String credentialToken) throws Exception {
+
+        LocalResourceIdentifier resource = LocalTransportUtil.getLocalResourceIdentifier(resourceId);
+        File file = new File(resource.getPath());
+
+        ResourceMetadata metadata = new ResourceMetadata();
+        metadata.setResourceSize(file.length());
+        metadata.setUpdateTime(file.lastModified());
+        return metadata;
+    }
+}
diff --git a/transport/local-transport/src/main/java/org/apache/airavata/mft/transport/local/LocalReceiver.java b/transport/local-transport/src/main/java/org/apache/airavata/mft/transport/local/LocalReceiver.java
new file mode 100644
index 0000000..3bdee70
--- /dev/null
+++ b/transport/local-transport/src/main/java/org/apache/airavata/mft/transport/local/LocalReceiver.java
@@ -0,0 +1,59 @@
+package org.apache.airavata.mft.transport.local;
+
+import org.apache.airavata.mft.core.ConnectorContext;
+import org.apache.airavata.mft.core.api.Connector;
+
+import java.io.FileInputStream;
+import java.io.OutputStream;
+
+public class LocalReceiver implements Connector {
+
+    private LocalResourceIdentifier resource;
+    @Override
+    public void init(String resourceId, String credentialToken) throws Exception {
+        this.resource = LocalTransportUtil.getLocalResourceIdentifier(resourceId);
+    }
+
+    @Override
+    public void destroy() {
+
+    }
+
+    @Override
+    public void startStream(ConnectorContext context) throws Exception {
+        System.out.println("Starting local receive");
+        FileInputStream fin = new FileInputStream(this.resource.getPath());
+
+        long fileSize = context.getMetadata().getResourceSize();
+
+        OutputStream outputStream = context.getStreamBuffer().getOutputStream();
+
+        byte[] buf = new byte[1024];
+        while (true) {
+            int bufSize = 0;
+
+            if (buf.length < fileSize) {
+                bufSize = buf.length;
+            } else {
+                bufSize = (int) fileSize;
+            }
+            bufSize = fin.read(buf, 0, bufSize);
+
+            if (bufSize < 0) {
+                break;
+            }
+
+            outputStream.write(buf, 0, bufSize);
+            outputStream.flush();
+
+            fileSize -= bufSize;
+            if (fileSize == 0L)
+                break;
+        }
+
+        fin.close();
+        outputStream.close();
+        //context.getStreamBuffer().close();
+        System.out.println("Completed local receive");
+    }
+}
diff --git a/transport/local-transport/src/main/java/org/apache/airavata/mft/transport/local/LocalResourceIdentifier.java b/transport/local-transport/src/main/java/org/apache/airavata/mft/transport/local/LocalResourceIdentifier.java
new file mode 100644
index 0000000..5e1c528
--- /dev/null
+++ b/transport/local-transport/src/main/java/org/apache/airavata/mft/transport/local/LocalResourceIdentifier.java
@@ -0,0 +1,14 @@
+package org.apache.airavata.mft.transport.local;
+
+public class LocalResourceIdentifier {
+
+    private String path;
+
+    public String getPath() {
+        return path;
+    }
+
+    public void setPath(String path) {
+        this.path = path;
+    }
+}
diff --git a/transport/local-transport/src/main/java/org/apache/airavata/mft/transport/local/LocalSender.java b/transport/local-transport/src/main/java/org/apache/airavata/mft/transport/local/LocalSender.java
new file mode 100644
index 0000000..bd2b366
--- /dev/null
+++ b/transport/local-transport/src/main/java/org/apache/airavata/mft/transport/local/LocalSender.java
@@ -0,0 +1,54 @@
+package org.apache.airavata.mft.transport.local;
+
+import org.apache.airavata.mft.core.ConnectorContext;
+import org.apache.airavata.mft.core.api.Connector;
+
+import java.io.FileOutputStream;
+import java.io.InputStream;
+
+public class LocalSender implements Connector {
+
+    private LocalResourceIdentifier resource;
+    @Override
+    public void init(String resourceId, String credentialToken) throws Exception {
+        this.resource = LocalTransportUtil.getLocalResourceIdentifier(resourceId);
+    }
+
+    @Override
+    public void destroy() {
+
+    }
+
+    @Override
+    public void startStream(ConnectorContext context) throws Exception {
+        System.out.println("Starting local send");
+        FileOutputStream fos = new FileOutputStream(this.resource.getPath());
+        long fileSize = context.getMetadata().getResourceSize();
+
+        InputStream inputStream = context.getStreamBuffer().getInputStream();
+
+        byte[] buf = new byte[1];
+        while (true) {
+            int bufSize = 0;
+
+            if (buf.length < fileSize) {
+                bufSize = buf.length;
+            } else {
+                bufSize = (int) fileSize;
+            }
+            bufSize = inputStream.read(buf, 0, bufSize);
+
+            if (bufSize < 0) {
+                break;
+            }
+
+            fos.write(buf, 0, bufSize);
+            fos.flush();
+
+            fileSize -= bufSize;
+            if (fileSize == 0L)
+                break;
+        }
+        System.out.println("Completed local send");
+    }
+}
diff --git a/transport/local-transport/src/main/java/org/apache/airavata/mft/transport/local/LocalTransportUtil.java b/transport/local-transport/src/main/java/org/apache/airavata/mft/transport/local/LocalTransportUtil.java
new file mode 100644
index 0000000..4de7302
--- /dev/null
+++ b/transport/local-transport/src/main/java/org/apache/airavata/mft/transport/local/LocalTransportUtil.java
@@ -0,0 +1,18 @@
+package org.apache.airavata.mft.transport.local;
+
+public class LocalTransportUtil {
+
+    public static LocalResourceIdentifier getLocalResourceIdentifier(String id) {
+        LocalResourceIdentifier identifier = new LocalResourceIdentifier();
+
+        switch (id) {
+            case "1":
+                identifier.setPath("/Users/dimuthu/data.csv");
+                return identifier;
+            case "2":
+                identifier.setPath("/Users/dimuthu/new.txt");
+                return identifier;
+        }
+        return null;
+    }
+}
diff --git a/transport/pom.xml b/transport/pom.xml
index 13af130..64266c5 100755
--- a/transport/pom.xml
+++ b/transport/pom.xml
@@ -34,6 +34,7 @@
     <packaging>pom</packaging>
     <modules>
         <module>scp-transport</module>
+        <module>local-transport</module>
     </modules>
 
 
diff --git a/transport/scp-transport/src/main/java/org/apache/airavata/mft/transport/scp/SCPReceiver.java b/transport/scp-transport/src/main/java/org/apache/airavata/mft/transport/scp/SCPReceiver.java
index a639d36..7dd4339 100644
--- a/transport/scp-transport/src/main/java/org/apache/airavata/mft/transport/scp/SCPReceiver.java
+++ b/transport/scp-transport/src/main/java/org/apache/airavata/mft/transport/scp/SCPReceiver.java
@@ -2,8 +2,8 @@ package org.apache.airavata.mft.transport.scp;
 
 import com.jcraft.jsch.ChannelExec;
 import com.jcraft.jsch.Session;
+import org.apache.airavata.mft.core.CircularStreamingBuffer;
 import org.apache.airavata.mft.core.ConnectorContext;
-import org.apache.airavata.mft.core.StreamBuffer;
 import org.apache.airavata.mft.core.api.Connector;
 
 import java.io.IOException;
@@ -35,7 +35,7 @@ public class SCPReceiver implements Connector {
         transferRemoteToStream(session, sshResourceIdentifier.getRemotePath(), context.getStreamBuffer());
     }
 
-    private void transferRemoteToStream(Session session, String from, StreamBuffer streamBuffer) throws Exception {
+    private void transferRemoteToStream(Session session, String from, CircularStreamingBuffer streamBuffer) throws Exception {
 
         OutputStream outputStream = streamBuffer.getOutputStream();
 
diff --git a/transport/scp-transport/src/main/java/org/apache/airavata/mft/transport/scp/SCPSender.java b/transport/scp-transport/src/main/java/org/apache/airavata/mft/transport/scp/SCPSender.java
index a1552ee..479bccf 100644
--- a/transport/scp-transport/src/main/java/org/apache/airavata/mft/transport/scp/SCPSender.java
+++ b/transport/scp-transport/src/main/java/org/apache/airavata/mft/transport/scp/SCPSender.java
@@ -3,8 +3,8 @@ package org.apache.airavata.mft.transport.scp;
 import com.jcraft.jsch.ChannelExec;
 import com.jcraft.jsch.JSchException;
 import com.jcraft.jsch.Session;
+import org.apache.airavata.mft.core.CircularStreamingBuffer;
 import org.apache.airavata.mft.core.ConnectorContext;
-import org.apache.airavata.mft.core.StreamBuffer;
 import org.apache.airavata.mft.core.api.Connector;
 
 import java.io.IOException;
@@ -34,7 +34,7 @@ public class SCPSender implements Connector {
         copyLocalToRemote(this.session, sshResourceIdentifier.getRemotePath(), context.getStreamBuffer(), context.getMetadata().getResourceSize());
     }
 
-    private void copyLocalToRemote(Session session, String to, StreamBuffer streamBuffer, long fileSize) throws JSchException, IOException {
+    private void copyLocalToRemote(Session session, String to, CircularStreamingBuffer streamBuffer, long fileSize) throws JSchException, IOException {
         System.out.println("Starting scp send");
         InputStream inputStream = streamBuffer.getInputStream();
 
@@ -93,6 +93,7 @@ public class SCPSender implements Connector {
             } else {
                 out.write(buf, 0, len); //out.flush();
                 totalWritten += len;
+                //System.out.println("Write " + totalWritten);
                 if (totalWritten == fileSize) {
                     break;
                 }