You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airavata.apache.org by sc...@apache.org on 2016/01/12 16:47:48 UTC
[05/24] airavata git commit: renaming the module

http://git-wip-us.apache.org/repos/asf/airavata/blob/1ad9ae5d/modules/data-manager/data-manager-core/src/main/java/org/apache/airavata/data/manager/core/remote/client/scp/SCPApiException.java
----------------------------------------------------------------------
diff --git a/modules/data-manager/data-manager-core/src/main/java/org/apache/airavata/data/manager/core/remote/client/scp/SCPApiException.java b/modules/data-manager/data-manager-core/src/main/java/org/apache/airavata/data/manager/core/remote/client/scp/SCPApiException.java
new file mode 100644
index 0000000..884b543
--- /dev/null
+++ b/modules/data-manager/data-manager-core/src/main/java/org/apache/airavata/data/manager/core/remote/client/scp/SCPApiException.java
@@ -0,0 +1,33 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+*/
+package org.apache.airavata.data.manager.core.remote.client.scp;
+
+public class SCPApiException extends Exception {
+
+    public SCPApiException(String message) {
+        super(message);
+    }
+
+    public SCPApiException(String message, Exception e) {
+        super(message, e);
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/airavata/blob/1ad9ae5d/modules/data-manager/data-manager-core/src/main/java/org/apache/airavata/data/manager/core/remote/client/scp/SCPStorageClient.java
----------------------------------------------------------------------
diff --git a/modules/data-manager/data-manager-core/src/main/java/org/apache/airavata/data/manager/core/remote/client/scp/SCPStorageClient.java b/modules/data-manager/data-manager-core/src/main/java/org/apache/airavata/data/manager/core/remote/client/scp/SCPStorageClient.java
new file mode 100644
index 0000000..db85c28
--- /dev/null
+++ b/modules/data-manager/data-manager-core/src/main/java/org/apache/airavata/data/manager/core/remote/client/scp/SCPStorageClient.java
@@ -0,0 +1,405 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+*/
+package org.apache.airavata.data.manager.core.remote.client.scp;
+
+import com.jcraft.jsch.*;
+import org.apache.airavata.data.manager.core.remote.client.RemoteStorageClient;
+import org.apache.airavata.model.data.transfer.LSEntryModel;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.*;
+import java.util.List;
+import java.util.Properties;
+import java.util.UUID;
+
+public class SCPStorageClient implements RemoteStorageClient {
+    private final static Logger logger = LoggerFactory.getLogger(SCPStorageClient.class);
+
+    private JSch jSch;
+    private Session session;
+
+    /**
+     * Constructor
+     * @param hostName
+     * @param port
+     * @param loginUsername
+     * @param password
+     * @throws JSchException
+     */
+    public SCPStorageClient(String hostName, int port, String loginUsername, String password) throws JSchException {
+        Properties config = new java.util.Properties();
+        config.put("StrictHostKeyChecking", "no");
+        jSch = new JSch();
+        jSch.addIdentity(loginUsername, password);
+        session = jSch.getSession(loginUsername, hostName, port);
+        session.setConfig(config);
+        session.connect();
+    }
+
+    /**
+     * Constructor
+     * @param hostName
+     * @param port
+     * @param loginUsername
+     * @param privateKey
+     * @param publicKey
+     * @param passPhrase
+     * @throws JSchException
+     */
+    public SCPStorageClient(String hostName, int port, String loginUsername, byte[] privateKey, byte[] publicKey,
+                            byte[] passPhrase) throws JSchException {
+        Properties config = new java.util.Properties();
+        config.put("StrictHostKeyChecking", "no");
+        jSch = new JSch();
+        jSch.addIdentity(UUID.randomUUID().toString(), privateKey, publicKey, passPhrase);
+        session = jSch.getSession(loginUsername, hostName, port);
+        session.setConfig(config);
+        session.connect();
+    }
+
+
+    /**
+     * Reads a remote file, write it to local temporary directory and returns a File
+     *
+     * @param filePath
+     * @return
+     * @throws Exception
+     */
+    @Override
+    public File readFile(String filePath) throws Exception {
+        if (!session.isConnected())
+            session.connect();
+
+        FileOutputStream fos;
+        String localFile = System.getProperty("java.io.tmpdir") + File.separator + UUID.randomUUID().toString();
+        String prefix = null;
+        if (new File(localFile).isDirectory()) {
+            prefix = localFile + File.separator;
+        }
+
+        // exec 'scp -f remotefile' remotely
+        String command = "scp -f " + filePath;
+        Channel channel = session.openChannel("exec");
+        ((ChannelExec) channel).setCommand(command);
+
+        StandardOutReader stdOutReader = new StandardOutReader();
+        ((ChannelExec) channel).setErrStream(stdOutReader.getStandardError());
+        // get I/O streams for remote scp
+        OutputStream out = channel.getOutputStream();
+        InputStream in = channel.getInputStream();
+
+        if (!channel.isClosed()) {
+            channel.connect();
+        }
+
+        byte[] buf = new byte[1024];
+
+        // send '\0'
+        buf[0] = 0;
+        out.write(buf, 0, 1);
+        out.flush();
+
+        while (true) {
+            int c = checkAck(in);
+            if (c != 'C') {
+                break;
+            }
+
+            // read '0644 '
+            in.read(buf, 0, 5);
+
+            long filesize = 0L;
+            while (true) {
+                if (in.read(buf, 0, 1) < 0) {
+                    // error
+                    break;
+                }
+                if (buf[0] == ' ') break;
+                filesize = filesize * 10L + (long) (buf[0] - '0');
+            }
+
+            String file = null;
+            for (int i = 0; ; i++) {
+                in.read(buf, i, 1);
+                if (buf[i] == (byte) 0x0a) {
+                    file = new String(buf, 0, i);
+                    break;
+                }
+            }
+
+            // send '\0'
+            buf[0] = 0;
+            out.write(buf, 0, 1);
+            out.flush();
+
+            // read a content of lfile
+            localFile = prefix == null ? localFile : prefix + file;
+            fos = new FileOutputStream(localFile);
+            int foo;
+            while (true) {
+                if (buf.length < filesize) foo = buf.length;
+                else foo = (int) filesize;
+                foo = in.read(buf, 0, foo);
+                if (foo < 0) {
+                    // error
+                    break;
+                }
+                fos.write(buf, 0, foo);
+                filesize -= foo;
+                if (filesize == 0L) break;
+            }
+            fos.close();
+            fos = null;
+
+            if (checkAck(in) != 0) {
+                String error = "Error transferring the file content";
+                logger.error(error);
+                throw new SCPApiException(error);
+            }
+
+            // send '\0'
+            buf[0] = 0;
+            out.write(buf, 0, 1);
+            out.flush();
+        }
+        stdOutReader.onOutput(channel);
+        if (stdOutReader.getStdErrorString().contains("scp:")) {
+            throw new SCPApiException(stdOutReader.getStdErrorString());
+        }
+
+        return new File(localFile);
+    }
+
+    /**
+     * Writes the source file in the local storage to specified path in the remote storage
+     *
+     * @param sourceFile
+     * @param filePath
+     * @return
+     * @throws Exception
+     */
+    @Override
+    public void writeFile(File sourceFile, String filePath) throws Exception {
+        if (!session.isConnected())
+            session.connect();
+
+        FileInputStream fis;
+        String localFile = sourceFile.getAbsolutePath();
+        boolean ptimestamp = true;
+
+        // exec 'scp -t rfile' remotely
+        String command = "scp " + (ptimestamp ? "-p" : "") + " -t " + filePath;
+        Channel channel = session.openChannel("exec");
+
+        StandardOutReader stdOutReader = new StandardOutReader();
+        ((ChannelExec) channel).setErrStream(stdOutReader.getStandardError());
+        ((ChannelExec) channel).setCommand(command);
+
+        // get I/O streams for remote scp
+        OutputStream out = channel.getOutputStream();
+        InputStream in = channel.getInputStream();
+
+        channel.connect();
+
+        if (checkAck(in) != 0) {
+            String error = "Error Reading input Stream";
+            logger.error(error);
+            throw new SCPApiException(error);
+        }
+
+        File _lfile = new File(localFile);
+
+        if (ptimestamp) {
+            command = "T" + (_lfile.lastModified() / 1000) + " 0";
+            // The access time should be sent here,
+            // but it is not accessible with JavaAPI ;-<
+            command += (" " + (_lfile.lastModified() / 1000) + " 0\n");
+            out.write(command.getBytes());
+            out.flush();
+            if (checkAck(in) != 0) {
+                String error = "Error Reading input Stream";
+                logger.error(error);
+                throw new SCPApiException(error);
+            }
+        }
+
+        // send "C0644 filesize filename", where filename should not include '/'
+        long filesize = _lfile.length();
+        command = "C0644 " + filesize + " ";
+        if (localFile.lastIndexOf('/') > 0) {
+            command += localFile.substring(localFile.lastIndexOf('/') + 1);
+        } else {
+            command += localFile;
+        }
+        command += "\n";
+        out.write(command.getBytes());
+        out.flush();
+        if (checkAck(in) != 0) {
+            String error = "Error Reading input Stream";
+            logger.error(error);
+            throw new SCPApiException(error);
+        }
+
+        // send a content of localFile
+        fis = new FileInputStream(localFile);
+        byte[] buf = new byte[1024];
+        while (true) {
+            int len = fis.read(buf, 0, buf.length);
+            if (len <= 0) break;
+            out.write(buf, 0, len); //out.flush();
+        }
+        fis.close();
+        fis = null;
+        // send '\0'
+        buf[0] = 0;
+        out.write(buf, 0, 1);
+        out.flush();
+        if (checkAck(in) != 0) {
+            String error = "Error Reading input Stream";
+            logger.error(error);
+            throw new SCPApiException(error);
+        }
+        out.close();
+        stdOutReader.onOutput(channel);
+
+
+        channel.disconnect();
+        if (stdOutReader.getStdErrorString().contains("scp:")) {
+            throw new SCPApiException(stdOutReader.getStdErrorString());
+        }
+    }
+
+    /**
+     * Returns a directory listing of the specified directory
+     *
+     * @param directoryPath
+     * @return
+     * @throws Exception
+     */
+    @Override
+    public List<LSEntryModel> getDirectoryListing(String directoryPath) throws Exception {
+        return null;
+    }
+
+    /**
+     * Move the specified file from source to destination within the same storage resource
+     *
+     * @param currentPath
+     * @param newPath
+     * @throws Exception
+     */
+    @Override
+    public void moveFile(String currentPath, String newPath) throws Exception {
+
+    }
+
+    /**
+     * @param sourcePath
+     * @param destinationPath
+     * @throws Exception
+     */
+    @Override
+    public void copyFile(String sourcePath, String destinationPath) throws Exception {
+
+    }
+
+    /**
+     * Rename file with the given name
+     *
+     * @param filePath
+     * @param newFileName
+     * @throws Exception
+     */
+    @Override
+    public void renameFile(String filePath, String newFileName) throws Exception {
+
+    }
+
+    /**
+     * Delete the specified file
+     *
+     * @param filePath
+     * @throws Exception
+     */
+    @Override
+    public void deleteFile(String filePath) throws Exception {
+
+    }
+
+    /**
+     * Create new directory in the specified file
+     *
+     * @param newDirPath
+     * @throws Exception
+     */
+    @Override
+    public void mkdir(String newDirPath) throws Exception {
+
+    }
+
+    /**
+     * Checks whether specified file exists in the remote storage system
+     *
+     * @param filePath
+     * @return
+     * @throws Exception
+     */
+    @Override
+    public boolean checkFileExists(String filePath) throws Exception {
+        return false;
+    }
+
+    /**
+     * Checks whether the given path is a directory
+     *
+     * @param filePath
+     * @return
+     * @throws Exception
+     */
+    @Override
+    public boolean checkIsDirectory(String filePath) throws Exception {
+        return false;
+    }
+
+    private int checkAck(InputStream in) throws IOException {
+        int b = in.read();
+        if (b == 0) return b;
+        if (b == -1) return b;
+
+        if (b == 1 || b == 2) {
+            StringBuffer sb = new StringBuffer();
+            int c;
+            do {
+                c = in.read();
+                sb.append((char) c);
+            }
+            while (c != '\n');
+            if (b == 1) { // error
+                System.out.print(sb.toString());
+            }
+            if (b == 2) { // fatal error
+                System.out.print(sb.toString());
+            }
+        }
+        return b;
+    }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/airavata/blob/1ad9ae5d/modules/data-manager/data-manager-core/src/main/java/org/apache/airavata/data/manager/core/remote/client/scp/StandardOutReader.java
----------------------------------------------------------------------
diff --git a/modules/data-manager/data-manager-core/src/main/java/org/apache/airavata/data/manager/core/remote/client/scp/StandardOutReader.java b/modules/data-manager/data-manager-core/src/main/java/org/apache/airavata/data/manager/core/remote/client/scp/StandardOutReader.java
new file mode 100644
index 0000000..f7c7d43
--- /dev/null
+++ b/modules/data-manager/data-manager-core/src/main/java/org/apache/airavata/data/manager/core/remote/client/scp/StandardOutReader.java
@@ -0,0 +1,86 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+*/
+package org.apache.airavata.data.manager.core.remote.client.scp;
+
+import com.jcraft.jsch.Channel;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+
+public class StandardOutReader implements CommandOutput {
+
+    private static final Logger logger = LoggerFactory.getLogger(StandardOutReader.class);
+    String stdOutputString = null;
+    ByteArrayOutputStream errorStream = new ByteArrayOutputStream();
+    private int exitCode;
+
+    public void onOutput(Channel channel) {
+        try {
+            StringBuffer pbsOutput = new StringBuffer("");
+            InputStream inputStream =  channel.getInputStream();
+            byte[] tmp = new byte[1024];
+            do {
+                while (inputStream.available() > 0) {
+                    int i = inputStream.read(tmp, 0, 1024);
+                    if (i < 0) break;
+                    pbsOutput.append(new String(tmp, 0, i));
+                }
+            } while (!channel.isClosed()) ;
+            String output = pbsOutput.toString();
+            this.setStdOutputString(output);
+        } catch (IOException e) {
+            logger.error(e.getMessage(), e);
+        }
+
+    }
+
+
+    public void exitCode(int code) {
+        System.out.println("Program exit code - " + code);
+        this.exitCode = code;
+    }
+
+    @Override
+    public int getExitCode() {
+        return exitCode;
+    }
+
+    public String getStdOutputString() {
+        return stdOutputString;
+    }
+
+    public void setStdOutputString(String stdOutputString) {
+        this.stdOutputString = stdOutputString;
+    }
+
+    public String getStdErrorString() {
+        return errorStream.toString();
+    }
+
+    public OutputStream getStandardError() {
+        return errorStream;
+    }
+}

http://git-wip-us.apache.org/repos/asf/airavata/blob/1ad9ae5d/modules/data-manager/data-manager-core/src/main/java/org/apache/airavata/data/manager/core/remote/client/sftp/SFTPStorageClient.java
----------------------------------------------------------------------
diff --git a/modules/data-manager/data-manager-core/src/main/java/org/apache/airavata/data/manager/core/remote/client/sftp/SFTPStorageClient.java b/modules/data-manager/data-manager-core/src/main/java/org/apache/airavata/data/manager/core/remote/client/sftp/SFTPStorageClient.java
new file mode 100644
index 0000000..c802498
--- /dev/null
+++ b/modules/data-manager/data-manager-core/src/main/java/org/apache/airavata/data/manager/core/remote/client/sftp/SFTPStorageClient.java
@@ -0,0 +1,209 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+*/
+package org.apache.airavata.data.manager.core.remote.client.sftp;
+
+import com.jcraft.jsch.ChannelSftp;
+import com.jcraft.jsch.JSch;
+import com.jcraft.jsch.JSchException;
+import com.jcraft.jsch.Session;
+import org.apache.airavata.data.manager.core.remote.client.RemoteStorageClient;
+import org.apache.airavata.model.data.transfer.LSEntryModel;
+import org.apache.airavata.model.data.transfer.LSEntryType;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.util.*;
+
+public class SFTPStorageClient implements RemoteStorageClient {
+    private final static Logger logger = LoggerFactory.getLogger(SFTPStorageClient.class);
+
+    private JSch jSch;
+    private Session session;
+    private ChannelSftp sftpChannel;
+    private final String hostName;
+
+    public SFTPStorageClient(String hostName, int port, String loginUsername, String password) throws JSchException {
+        this.hostName = hostName;
+        Properties config = new java.util.Properties();
+        config.put("StrictHostKeyChecking", "no");
+        jSch = new JSch();
+        jSch.addIdentity(loginUsername, password);
+        session = jSch.getSession(loginUsername, hostName, port);
+        session.setConfig(config);
+        session.connect();
+        sftpChannel = (ChannelSftp) session.openChannel("sftp");
+    }
+
+    public SFTPStorageClient(String hostName, int port, String loginUsername, byte[] privateKey, byte[] publicKey,
+                            byte[] passPhrase) throws JSchException {
+        this.hostName = hostName;
+        Properties config = new java.util.Properties();
+        config.put("StrictHostKeyChecking", "no");
+        jSch = new JSch();
+        jSch.addIdentity(UUID.randomUUID().toString(), privateKey, publicKey, passPhrase);
+        session = jSch.getSession(loginUsername, hostName, port);
+        session.setConfig(config);
+        session.connect();
+        sftpChannel = (ChannelSftp) session.openChannel("sftp");
+    }
+
+    /**
+     * Reads a remote file, write it to local temporary directory and returns a file pointer to it
+     *
+     * @param filePath
+     * @return
+     * @throws Exception
+     */
+    @Override
+    public File readFile(String filePath) throws Exception {
+        String localFile = System.getProperty("java.io.tmpdir") + File.separator + UUID.randomUUID().toString();
+        return null;
+    }
+
+    /**
+     * Writes the source file in the local storage to specified path in the remote storage
+     *
+     * @param sourceFile
+     * @param filePath
+     * @return
+     * @throws Exception
+     */
+    @Override
+    public void writeFile(File sourceFile, String filePath) throws Exception {
+
+    }
+
+    /**
+     * Returns a directory listing of the specified directory
+     *
+     * @param directoryPath
+     * @return
+     * @throws Exception
+     */
+    @Override
+    public List<LSEntryModel> getDirectoryListing(String directoryPath) throws Exception {
+        if(directoryPath.endsWith(File.separator)){
+            directoryPath = directoryPath.substring(0, directoryPath.length() -1);
+        }
+        final String finalDirPath =  directoryPath;
+        //channel may get timeout
+        if(sftpChannel.isClosed()){
+            sftpChannel.connect();
+        }
+        sftpChannel.cd(directoryPath);
+        Vector<ChannelSftp.LsEntry> lsEntryVector = sftpChannel.ls(directoryPath);
+        ArrayList<LSEntryModel> fileNodeList = new ArrayList<>();
+        lsEntryVector.stream().forEach(lsEntry -> {
+            LSEntryModel fileNode = new LSEntryModel();
+            fileNode.setName(lsEntry.getFilename());
+            fileNode.setPath(finalDirPath + File.separator + lsEntry.getFilename());
+            fileNode.setStorageHostName(hostName);
+            fileNode.setSize(lsEntry.getAttrs().getSize());
+            if(lsEntry.getAttrs().isDir())
+                fileNode.setType(LSEntryType.DIRECTORY);
+            else
+                fileNode.setType(LSEntryType.FILE);
+            fileNodeList.add(fileNode);
+        });
+        return fileNodeList;
+    }
+
+    /**
+     * Move the specified file from source to destination within the same storage resource
+     *
+     * @param currentPath
+     * @param newPath
+     * @throws Exception
+     */
+    @Override
+    public void moveFile(String currentPath, String newPath) throws Exception {
+
+    }
+
+    /**
+     * @param sourcePath
+     * @param destinationPath
+     * @throws Exception
+     */
+    @Override
+    public void copyFile(String sourcePath, String destinationPath) throws Exception {
+
+    }
+
+    /**
+     * Rename file with the given name
+     *
+     * @param filePath
+     * @param newFileName
+     * @throws Exception
+     */
+    @Override
+    public void renameFile(String filePath, String newFileName) throws Exception {
+
+    }
+
+    /**
+     * Delete the specified file
+     *
+     * @param filePath
+     * @throws Exception
+     */
+    @Override
+    public void deleteFile(String filePath) throws Exception {
+
+    }
+
+    /**
+     * Create new directory in the specified file
+     *
+     * @param newDirPath
+     * @throws Exception
+     */
+    @Override
+    public void mkdir(String newDirPath) throws Exception {
+
+    }
+
+    /**
+     * Checks whether specified file exists in the remote storage system
+     *
+     * @param filePath
+     * @return
+     * @throws Exception
+     */
+    @Override
+    public boolean checkFileExists(String filePath) throws Exception {
+        return false;
+    }
+
+    /**
+     * Checks whether the given path is a directory
+     *
+     * @param filePath
+     * @return
+     * @throws Exception
+     */
+    @Override
+    public boolean checkIsDirectory(String filePath) throws Exception {
+        return false;
+    }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/airavata/blob/1ad9ae5d/modules/data-manager/data-manager-core/src/main/java/org/apache/airavata/data/manager/core/utils/FileManagerConstants.java
----------------------------------------------------------------------
diff --git a/modules/data-manager/data-manager-core/src/main/java/org/apache/airavata/data/manager/core/utils/FileManagerConstants.java b/modules/data-manager/data-manager-core/src/main/java/org/apache/airavata/data/manager/core/utils/FileManagerConstants.java
new file mode 100644
index 0000000..a182037
--- /dev/null
+++ b/modules/data-manager/data-manager-core/src/main/java/org/apache/airavata/data/manager/core/utils/FileManagerConstants.java
@@ -0,0 +1,33 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+*/
+package org.apache.airavata.data.manager.core.utils;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class FileManagerConstants {
+    private final static Logger logger = LoggerFactory.getLogger(FileManagerConstants.class);
+
+    public static String MONGODB_HOST = "mongodb.host";
+    public static String MONGODB_PORT = "mongodb.port";
+    public static String MONGODB_DB_NAME = "mongodb.db.name";
+
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/airavata/blob/1ad9ae5d/modules/data-manager/data-manager-core/src/main/java/org/apache/airavata/data/manager/core/utils/FileManagerProperties.java
----------------------------------------------------------------------
diff --git a/modules/data-manager/data-manager-core/src/main/java/org/apache/airavata/data/manager/core/utils/FileManagerProperties.java b/modules/data-manager/data-manager-core/src/main/java/org/apache/airavata/data/manager/core/utils/FileManagerProperties.java
new file mode 100644
index 0000000..1e5e45b
--- /dev/null
+++ b/modules/data-manager/data-manager-core/src/main/java/org/apache/airavata/data/manager/core/utils/FileManagerProperties.java
@@ -0,0 +1,55 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+*/
+package org.apache.airavata.data.manager.core.utils;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.Properties;
+
+public class FileManagerProperties {
+    private final static Logger logger = LoggerFactory.getLogger(FileManagerProperties.class);
+
+    private static FileManagerProperties instance;
+
+    private Properties properties;
+
+    private FileManagerProperties() throws IOException {
+        properties = new Properties();
+        properties.load(FileManagerProperties.class.getClassLoader().getResourceAsStream("file-manager.properties"));
+    }
+
+    public static FileManagerProperties getInstance() throws IOException {
+        if(instance == null){
+            instance = new FileManagerProperties();
+        }
+        return instance;
+    }
+
+    public String getProperty(String field, String defaultVal){
+        String returnVal = properties.getProperty(field);
+        if(returnVal == null)
+            return defaultVal;
+        else
+            return returnVal;
+    }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/airavata/blob/1ad9ae5d/modules/data-manager/data-manager-core/src/main/resources/file-manager.properties
----------------------------------------------------------------------
diff --git a/modules/data-manager/data-manager-core/src/main/resources/file-manager.properties b/modules/data-manager/data-manager-core/src/main/resources/file-manager.properties
new file mode 100644
index 0000000..ad4157e
--- /dev/null
+++ b/modules/data-manager/data-manager-core/src/main/resources/file-manager.properties
@@ -0,0 +1,24 @@
+#
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+
+mongodb.host=localhost
+mongodb.port=27017
+mongodb.db.name=file-manager-db
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/airavata/blob/1ad9ae5d/modules/data-manager/data-manager-core/src/main/resources/gridftp-client.properties
----------------------------------------------------------------------
diff --git a/modules/data-manager/data-manager-core/src/main/resources/gridftp-client.properties b/modules/data-manager/data-manager-core/src/main/resources/gridftp-client.properties
new file mode 100644
index 0000000..7d8a757
--- /dev/null
+++ b/modules/data-manager/data-manager-core/src/main/resources/gridftp-client.properties
@@ -0,0 +1,30 @@
+trustedCertsFile=/Users/supun/Work/airavata-sandbox/grid-tools/certificates/
+myproxyServer=myproxy.teragrid.org
+myproxy_lifetime=17280000
+myproxyUserName=
+myproxyPasswd=
+myproxyPort=7512
+
+
+testing.host=trestles
+#testing.host=stampede
+#testing.host=lonestar
+
+## TACC Lonestar
+lonestar.gridftp.endpoint=gsiftp://gridftp1.ls4.tacc.utexas.edu:2811/
+
+## TACC Stampede
+stampede.gridftp.endpoint=gsiftp://gridftp.stampede.tacc.utexas.edu:2811/
+
+## SDSC Trestles
+trestles.gridftp.endpoint=gsiftp://trestles.sdsc.edu:2811/
+
+gridftpserverSource=gsiftp://trestles-dm.sdsc.xsede.org:2811/
+gridftpSourcePath=/oasis/projects/nsf/sds128/ogce/file-transfer-tests/source/sample_wrfout.netcdf
+gridftpUploadingFilePath = /oasis/projects/nsf/sds128/ogce/file-transfer-tests/source/dummy1
+
+#gridftpserverDest=gsiftp://trestles-dm.sdsc.xsede.org:2811/
+#gridftpDestPath=/oasis/projects/nsf/sds128/ogce/file-transfer-tests/gf
+
+gridftpserverDest=gsiftp://gridftp.stampede.tacc.utexas.edu:2811/
+gridftpDestPath=/scratch/01437/ogce/file-transfer-tests/dest/xx
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/airavata/blob/1ad9ae5d/modules/data-manager/data-manager-core/src/test/java/org/apache/airavata/data/manager/core/db/dao/FileTransferRequestDaoTest.java
----------------------------------------------------------------------
diff --git a/modules/data-manager/data-manager-core/src/test/java/org/apache/airavata/data/manager/core/db/dao/FileTransferRequestDaoTest.java b/modules/data-manager/data-manager-core/src/test/java/org/apache/airavata/data/manager/core/db/dao/FileTransferRequestDaoTest.java
new file mode 100644
index 0000000..327a603
--- /dev/null
+++ b/modules/data-manager/data-manager-core/src/test/java/org/apache/airavata/data/manager/core/db/dao/FileTransferRequestDaoTest.java
@@ -0,0 +1,45 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+*/
+package org.apache.airavata.data.manager.core.db.dao;
+
+import org.apache.airavata.model.data.transfer.FileTransferRequestModel;
+import org.junit.Ignore;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+
+@Ignore
+public class FileTransferRequestDaoTest {
+    private final static Logger logger = LoggerFactory.getLogger(FileTransferRequestDaoTest.class);
+
+    @Test
+    public void testFileTransferRequestDao() throws IOException {
+        FileTransferRequestModel fileTransferRequestModel = new FileTransferRequestModel();
+        fileTransferRequestModel.setSrcHostCredToken("djkalbsbdaslfbalsfbslf");
+        fileTransferRequestModel.setSrcFilePath("test-file-path");
+        FileTransferRequestDao fileTransferRequestDao = new FileTransferRequestDao();
+        String transferId = fileTransferRequestDao.createFileTransferRequest(fileTransferRequestModel);
+        fileTransferRequestModel = fileTransferRequestDao.getFileTransferRequest(transferId);
+        System.out.println("Transfer Id:" + fileTransferRequestModel.getTransferId());
+    }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/airavata/blob/1ad9ae5d/modules/data-manager/data-manager-core/src/test/java/org/apache/airavata/data/manager/core/remote/client/gridftp/CertFileReadTest.java
----------------------------------------------------------------------
diff --git a/modules/data-manager/data-manager-core/src/test/java/org/apache/airavata/data/manager/core/remote/client/gridftp/CertFileReadTest.java b/modules/data-manager/data-manager-core/src/test/java/org/apache/airavata/data/manager/core/remote/client/gridftp/CertFileReadTest.java
new file mode 100644
index 0000000..26931aa
--- /dev/null
+++ b/modules/data-manager/data-manager-core/src/test/java/org/apache/airavata/data/manager/core/remote/client/gridftp/CertFileReadTest.java
@@ -0,0 +1,126 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+*/
+package org.apache.airavata.data.manager.core.remote.client.gridftp;
+
+import junit.framework.Assert;
+import junit.framework.TestCase;
+import org.globus.gsi.SigningPolicy;
+import org.globus.gsi.SigningPolicyParser;
+import org.globus.gsi.util.CertificateIOUtil;
+import org.globus.util.GlobusResource;
+import org.junit.Ignore;
+import org.junit.Test;
+
+import javax.security.auth.x500.X500Principal;
+import java.io.FileInputStream;
+import java.io.InputStreamReader;
+import java.io.Reader;
+import java.security.MessageDigest;
+import java.security.cert.CertificateFactory;
+import java.security.cert.X509Certificate;
+import java.util.Map;
+
+
+@Ignore("This test case used to debug JGlobus-102. No need to run this test with other gridftp tests.")
+public class CertFileReadTest extends TestCase {
+
+    private static MessageDigest md5;
+
+    private static String CERT_FILE_LOCATION = "/Users/supun/Work/airavata-sandbox/grid-tools/certificates/";
+
+    @Test
+    public void testCertFileRead() throws Exception {
+
+        String path1 = CERT_FILE_LOCATION + "ffc3d59b";
+        String path2 = CERT_FILE_LOCATION + "e5cc84c2";
+
+
+        GlobusResource globusResource1 = new GlobusResource(path1 + ".signing_policy");
+        GlobusResource globusResource2 = new GlobusResource(path2 + ".signing_policy");
+
+        // ===== Testing globusResource1 - This should pass (cos no DC components) ================ //
+        X509Certificate crt1 = readCertificate(path1 + ".0");
+        X500Principal policySubjectCert1 = getPrincipal(globusResource1);
+
+        String certHash1 = CertificateIOUtil.nameHash(crt1.getSubjectX500Principal());
+        String principalHash1 = CertificateIOUtil.nameHash(policySubjectCert1);
+
+        System.out.println("======== Printing hashes for 1 ================");
+        System.out.println(certHash1);
+        System.out.println(principalHash1);
+
+        Assert.assertEquals("Certificate hash value does not match with the hash value generated using principal name.",
+                certHash1, principalHash1);
+
+        // ===== Testing globusResource1 - This should fail (cos we have DC components) ================ //
+        X509Certificate crt2 = readCertificate(path2 + ".0");
+        X500Principal policySubjectCert2 = getPrincipal(globusResource2);
+
+        String certHash2 = CertificateIOUtil.nameHash(crt2.getSubjectX500Principal());
+        String principalHash2 = CertificateIOUtil.nameHash(policySubjectCert2);
+
+        System.out.println("======== Printing hashes for 2 ================");
+        System.out.println(certHash2);
+        System.out.println(principalHash2);
+
+        Assert.assertEquals("Certificate hash value does not match with the hash value generated using principal name.",
+                certHash2, principalHash2);
+    }
+
+    private X500Principal getPrincipal(GlobusResource globusResource) throws Exception{
+
+        SigningPolicyParser parser = new SigningPolicyParser();
+
+        Reader reader = new InputStreamReader(globusResource.getInputStream());
+
+        Map<X500Principal, SigningPolicy> policies = parser.parse(reader);
+
+        return policies.keySet().iterator().next();
+
+    }
+
+    private X509Certificate readCertificate(String certPath) {
+        try {
+            FileInputStream fr = new FileInputStream(certPath);
+            CertificateFactory cf =
+                    CertificateFactory.getInstance("X509");
+            X509Certificate crt = (X509Certificate)
+                    cf.generateCertificate(fr);
+            System.out.println("Read certificate:");
+            System.out.println("\tCertificate for: " +
+                    crt.getSubjectDN());
+            System.out.println("\tCertificate issued by: " +
+                    crt.getIssuerDN());
+            System.out.println("\tCertificate is valid from " +
+                    crt.getNotBefore() + " to " + crt.getNotAfter());
+            System.out.println("\tCertificate SN# " +
+                    crt.getSerialNumber());
+            System.out.println("\tGenerated with " +
+                    crt.getSigAlgName());
+
+            return crt;
+        } catch (Exception e) {
+            e.printStackTrace();
+        }
+
+        return null;
+    }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/airavata/blob/1ad9ae5d/modules/data-manager/data-manager-core/src/test/java/org/apache/airavata/data/manager/core/remote/client/gridftp/FileTransferTest.java
----------------------------------------------------------------------
diff --git a/modules/data-manager/data-manager-core/src/test/java/org/apache/airavata/data/manager/core/remote/client/gridftp/FileTransferTest.java b/modules/data-manager/data-manager-core/src/test/java/org/apache/airavata/data/manager/core/remote/client/gridftp/FileTransferTest.java
new file mode 100644
index 0000000..ff482c4
--- /dev/null
+++ b/modules/data-manager/data-manager-core/src/test/java/org/apache/airavata/data/manager/core/remote/client/gridftp/FileTransferTest.java
@@ -0,0 +1,168 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+*/
+package org.apache.airavata.data.manager.core.remote.client.gridftp;
+
+
+import junit.framework.Assert;
+import junit.framework.TestCase;
+import org.apache.airavata.data.manager.core.remote.client.gridftp.myproxy.SecurityContext;
+import org.apache.log4j.BasicConfigurator;
+import org.apache.log4j.Level;
+import org.apache.log4j.Logger;
+import org.ietf.jgss.GSSCredential;
+import org.junit.Ignore;
+
+import java.io.*;
+import java.net.URI;
+
+
+@Ignore
+public class FileTransferTest extends TestCase {
+
+    private GSSCredential gssCredential;
+
+    private ExecutionContext executionContext;
+
+    private static final Logger log = Logger.getLogger(FileTransferTest.class);
+
+
+    public void setUp() throws Exception {
+
+        String userName = System.getProperty("myproxy.user");
+        String password = System.getProperty("myproxy.password");
+
+        SecurityContext context = null;
+
+        if (userName == null || password == null || userName.trim().equals("") || password.trim().equals("")) {
+            log.error("myproxy.user and myproxy.password system properties are not set. Example :- " +
+                    "> mvn clean install -Dmyproxy.user=u1 -Dmyproxy.password=xxx");
+
+            Assert.fail("Please set myproxy.user and myproxy.password system properties.");
+
+        } else {
+            context = new SecurityContext(userName, password);
+        }
+
+        log.info("Using my proxy user name - " + userName);
+
+        BasicConfigurator.configure();
+        Logger logger = Logger.getLogger("GridFTPClient");
+        Level lev = Level.toLevel("DEBUG");
+        logger.setLevel(lev);
+
+
+        context.login();
+        executionContext = new ExecutionContext();
+
+
+        String targeterp = executionContext.getGridFTPServerDestination();
+        String remoteDestFile = executionContext.getDestinationDataLocation();
+
+        URI dirLocation = GridFTP.createGsiftpURI(targeterp,
+                remoteDestFile.substring(0, remoteDestFile.lastIndexOf("/")));
+        gssCredential = context.getGssCredential();
+        System.out.println(dirLocation);
+
+    }
+
+    public void testMakeDir() throws Exception {
+
+        String targetErp = executionContext.getGridFTPServerDestination();
+        String remoteDestinationFile = executionContext.getDestinationDataLocation();
+
+        URI dirLocation = GridFTP.createGsiftpURI(targetErp,
+                remoteDestinationFile.substring(0, remoteDestinationFile.lastIndexOf("/")));
+
+        GridFTP ftp = new GridFTP();
+        ftp.makeDir(dirLocation, gssCredential);
+
+        Assert.assertTrue(ftp.exists(dirLocation, gssCredential));
+
+    }
+
+    public void testTransferData() throws Exception {
+
+        String sourceERP = executionContext.getGridFTPServerSource();
+        String remoteSrcFile = executionContext.getSourceDataLocation();
+
+        String targetErp = executionContext.getGridFTPServerDestination();
+        String remoteDestinationFile = executionContext.getDestinationDataLocation();
+
+        URI srcURI = GridFTP.createGsiftpURI(sourceERP, remoteSrcFile);
+        URI destURI = GridFTP.createGsiftpURI(targetErp, remoteDestinationFile);
+
+        GridFTP ftp = new GridFTP();
+        ftp.transfer(srcURI, destURI, gssCredential, true);
+
+        Assert.assertTrue(ftp.exists(destURI, gssCredential));
+
+    }
+
+    public void testDownloadFile() throws Exception {
+
+        String fileName = "./downloaded";
+
+        File deleteFile = new File(fileName);
+
+        if (deleteFile.exists()) {
+            if (!deleteFile.delete())
+                throw new RuntimeException("Unable to delete file " + fileName);
+        }
+
+        File f = new File(fileName);
+
+        GridFTP ftp = new GridFTP();
+        ftp.downloadFile(executionContext.getSourceDataFileUri(),
+                gssCredential, f);
+
+        Assert.assertTrue(f.exists());
+
+    }
+
+    public void testFileExists() throws Exception {
+
+        GridFTP ftp = new GridFTP();
+        Assert.assertTrue(ftp.exists(executionContext.getSourceDataFileUri(), gssCredential));
+    }
+
+    public void testUpdateFile() throws Exception {
+
+        String currentDir = System.getProperty("projectDirectory");
+
+        if (currentDir == null)
+            currentDir = "src/test/resources";
+        else
+            currentDir = currentDir + "/src/test/resources";
+
+        String file = currentDir + "/dummy";
+
+        System.out.println("File to upload is " + file);
+
+        File fileToUpload = new File(file);
+
+        Assert.assertTrue(fileToUpload.canRead());
+
+        GridFTP ftp = new GridFTP();
+        ftp.updateFile(executionContext.getUploadingFilePathUri(), gssCredential, fileToUpload);
+
+        Assert.assertTrue(ftp.exists(executionContext.getUploadingFilePathUri(), gssCredential));
+    }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/airavata/blob/1ad9ae5d/modules/data-manager/data-manager-core/src/test/java/org/apache/airavata/data/manager/core/remote/client/http/HTTPStorageClientTest.java
----------------------------------------------------------------------
diff --git a/modules/data-manager/data-manager-core/src/test/java/org/apache/airavata/data/manager/core/remote/client/http/HTTPStorageClientTest.java b/modules/data-manager/data-manager-core/src/test/java/org/apache/airavata/data/manager/core/remote/client/http/HTTPStorageClientTest.java
new file mode 100644
index 0000000..9190a93
--- /dev/null
+++ b/modules/data-manager/data-manager-core/src/test/java/org/apache/airavata/data/manager/core/remote/client/http/HTTPStorageClientTest.java
@@ -0,0 +1,45 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+*/
+package org.apache.airavata.data.manager.core.remote.client.http;
+
+import junit.framework.Assert;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+
+public class HTTPStorageClientTest {
+    private final static Logger logger = LoggerFactory.getLogger(HTTPStorageClientTest.class);
+
+    @Test
+    public void testHTTPStorageClient(){
+        try {
+            HTTPStorageClient httpStorageClient = new HTTPStorageClient(HTTPStorageClient.Protocol.HTTPS,
+                    "www.google.lk", 443);
+            File file = httpStorageClient.readFile("/");
+            Assert.assertTrue(file.exists());
+        } catch (Exception e) {
+            e.printStackTrace();
+            Assert.fail();
+        }
+    }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/airavata/blob/1ad9ae5d/modules/data-manager/data-manager-core/src/test/java/org/apache/airavata/data/manager/core/remote/client/scp/SCPStorageClientTest.java
----------------------------------------------------------------------
diff --git a/modules/data-manager/data-manager-core/src/test/java/org/apache/airavata/data/manager/core/remote/client/scp/SCPStorageClientTest.java b/modules/data-manager/data-manager-core/src/test/java/org/apache/airavata/data/manager/core/remote/client/scp/SCPStorageClientTest.java
new file mode 100644
index 0000000..b5773e2
--- /dev/null
+++ b/modules/data-manager/data-manager-core/src/test/java/org/apache/airavata/data/manager/core/remote/client/scp/SCPStorageClientTest.java
@@ -0,0 +1,55 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+*/
+package org.apache.airavata.data.manager.core.remote.client.scp;
+
+import org.apache.commons.io.IOUtils;
+import org.junit.Ignore;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.FileInputStream;
+
+@Ignore
+public class SCPStorageClientTest {
+    private final static Logger logger = LoggerFactory.getLogger(SCPStorageClientTest.class);
+
+    @Test
+    public void testSCPStorageClient() throws Exception {
+        File privateKey = new File("/Users/supun/.ssh/id_rsa");
+        byte[] privateKeyBytes = IOUtils.toByteArray(new FileInputStream(privateKey));
+
+        File publicKey = new File("/Users/supun/.ssh/id_rsa.pub");
+        byte[] publicKeyBytes = IOUtils.toByteArray(new FileInputStream(publicKey));
+
+        String passPhrase = "";
+        byte[] passPhraseBytes = passPhrase.getBytes();
+
+        SCPStorageClient scpStorageClient = new SCPStorageClient("gw75.iu.xsede.org", 22, "pga", privateKeyBytes,
+                publicKeyBytes, passPhraseBytes);
+        File file = scpStorageClient.readFile("/var/www/portals/gateway-user-data/testdrive/test.txt");
+        System.out.println("File exists ? " + file.exists());
+        scpStorageClient.writeFile(file, "/var/www/portals/gateway-user-data/testdrive/test2.txt");
+        file.delete();
+        System.out.println("File exists ? " + file.exists());
+    }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/airavata/blob/1ad9ae5d/modules/data-manager/data-manager-core/src/test/resources/airavata-myproxy-client.properties
----------------------------------------------------------------------
diff --git a/modules/data-manager/data-manager-core/src/test/resources/airavata-myproxy-client.properties b/modules/data-manager/data-manager-core/src/test/resources/airavata-myproxy-client.properties
new file mode 100644
index 0000000..407f09d
--- /dev/null
+++ b/modules/data-manager/data-manager-core/src/test/resources/airavata-myproxy-client.properties
@@ -0,0 +1,28 @@
+#Download trusted certificates from - https://software.xsede.org/security/xsede-certs.tar.gz
+trustedCertsFile=../certificates
+myproxyServer=myproxy.teragrid.org
+myproxy_lifetime=17280000
+myproxyPort=7512
+
+testing.host=trestles
+#testing.host=stampede
+#testing.host=lonestar
+
+## TACC Lonestar
+lonestar.gridftp.endpoint=gsiftp://gridftp1.ls4.tacc.utexas.edu:2811/
+
+## TACC Stampede
+stampede.gridftp.endpoint=gsiftp://gridftp.stampede.tacc.utexas.edu:2811/
+
+## SDSC Trestles
+trestles.gridftp.endpoint=gsiftp://trestles.sdsc.edu:2811/
+
+gridftpserverSource=gsiftp://trestles-dm.sdsc.xsede.org:2811/
+gridftpSourcePath=/oasis/projects/nsf/sds128/ogce/file-transfer-tests/source/sample_wrfout.netcdf
+gridftpUploadingFilePath = /oasis/projects/nsf/sds128/ogce/file-transfer-tests/source/dummy1
+
+#gridftpserverDest=gsiftp://trestles-dm.sdsc.xsede.org:2811/
+#gridftpDestPath=/oasis/projects/nsf/sds128/ogce/file-transfer-tests/gf
+
+gridftpserverDest=gsiftp://gridftp.stampede.tacc.utexas.edu:2811/
+gridftpDestPath=/scratch/01437/ogce/file-transfer-tests/dest/xx
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/airavata/blob/1ad9ae5d/modules/data-manager/data-manager-core/src/test/resources/dummy
----------------------------------------------------------------------
diff --git a/modules/data-manager/data-manager-core/src/test/resources/dummy b/modules/data-manager/data-manager-core/src/test/resources/dummy
new file mode 100644
index 0000000..e69de29

http://git-wip-us.apache.org/repos/asf/airavata/blob/1ad9ae5d/modules/data-manager/data-manager-cpi/pom.xml
----------------------------------------------------------------------
diff --git a/modules/data-manager/data-manager-cpi/pom.xml b/modules/data-manager/data-manager-cpi/pom.xml
new file mode 100644
index 0000000..ef08821
--- /dev/null
+++ b/modules/data-manager/data-manager-cpi/pom.xml
@@ -0,0 +1,30 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <parent>
+        <artifactId>data-manager</artifactId>
+        <groupId>org.apache.airavata</groupId>
+        <version>0.16-SNAPSHOT</version>
+        <relativePath>../pom.xml</relativePath>
+    </parent>
+
+    <modelVersion>4.0.0</modelVersion>
+    <artifactId>data-manager-cpi</artifactId>
+    <packaging>jar</packaging>
+    <name>Airavata File Manager CPI</name>
+    <url>http://airavata.apache.org/</url>
+
+    <dependencies>
+        <dependency>
+            <groupId>org.apache.airavata</groupId>
+            <artifactId>airavata-data-models</artifactId>
+            <version>${project.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.airavata</groupId>
+            <artifactId>airavata-commons</artifactId>
+            <version>${project.version}</version>
+        </dependency>
+    </dependencies>
+</project>
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/airavata/blob/1ad9ae5d/modules/data-manager/data-manager-cpi/src/main/java/org/apache/airavata/data/manager/cpi/DataManager.java
----------------------------------------------------------------------
diff --git a/modules/data-manager/data-manager-cpi/src/main/java/org/apache/airavata/data/manager/cpi/DataManager.java b/modules/data-manager/data-manager-cpi/src/main/java/org/apache/airavata/data/manager/cpi/DataManager.java
new file mode 100644
index 0000000..47965f6
--- /dev/null
+++ b/modules/data-manager/data-manager-cpi/src/main/java/org/apache/airavata/data/manager/cpi/DataManager.java
@@ -0,0 +1,42 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+package org.apache.airavata.data.manager.cpi;
+
+public interface DataManager {
+
+    /**
+     * Return file transfer service instance
+     * @return
+     */
+    FileTransferService getFileTransferService() throws DataManagerException;
+
+    /**
+     * Return replica catalog service instance
+     * @return
+     */
+    ReplicaCatalogService getReplicaCatalogService() throws DataManagerException;
+
+    /**
+     * Return metadata catalog service
+     * @return
+     */
+    MetadataCatalogService getMetadataCatalogService() throws DataManagerException;
+}

http://git-wip-us.apache.org/repos/asf/airavata/blob/1ad9ae5d/modules/data-manager/data-manager-cpi/src/main/java/org/apache/airavata/data/manager/cpi/DataManagerConstants.java
----------------------------------------------------------------------
diff --git a/modules/data-manager/data-manager-cpi/src/main/java/org/apache/airavata/data/manager/cpi/DataManagerConstants.java b/modules/data-manager/data-manager-cpi/src/main/java/org/apache/airavata/data/manager/cpi/DataManagerConstants.java
new file mode 100644
index 0000000..c2407a5
--- /dev/null
+++ b/modules/data-manager/data-manager-cpi/src/main/java/org/apache/airavata/data/manager/cpi/DataManagerConstants.java
@@ -0,0 +1,32 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+*/
+package org.apache.airavata.data.manager.cpi;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class DataManagerConstants {
+    private final static Logger logger = LoggerFactory.getLogger(DataManagerConstants.class);
+
+    public static String AIRAVATA_FILE_ID_PREFIX = "airavata-file://";
+    public static String AIRAVATA_COLLECTION_ID_PREFIX = "airavata-collection://";
+    public static final String AIRAVATA_METADATA_ID_PREFIX = "airavata-metadata://";
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/airavata/blob/1ad9ae5d/modules/data-manager/data-manager-cpi/src/main/java/org/apache/airavata/data/manager/cpi/DataManagerException.java
----------------------------------------------------------------------
diff --git a/modules/data-manager/data-manager-cpi/src/main/java/org/apache/airavata/data/manager/cpi/DataManagerException.java b/modules/data-manager/data-manager-cpi/src/main/java/org/apache/airavata/data/manager/cpi/DataManagerException.java
new file mode 100644
index 0000000..b1c1cb8
--- /dev/null
+++ b/modules/data-manager/data-manager-cpi/src/main/java/org/apache/airavata/data/manager/cpi/DataManagerException.java
@@ -0,0 +1,35 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.airavata.data.manager.cpi;
+
+public class DataManagerException extends Exception{
+
+    public DataManagerException(Throwable e) {
+        super(e);
+    }
+
+    public DataManagerException(String message) {
+        super(message, null);
+    }
+
+    public DataManagerException(String message, Throwable e) {
+        super(message, e);
+    }
+}

http://git-wip-us.apache.org/repos/asf/airavata/blob/1ad9ae5d/modules/data-manager/data-manager-cpi/src/main/java/org/apache/airavata/data/manager/cpi/FileTransferService.java
----------------------------------------------------------------------
diff --git a/modules/data-manager/data-manager-cpi/src/main/java/org/apache/airavata/data/manager/cpi/FileTransferService.java b/modules/data-manager/data-manager-cpi/src/main/java/org/apache/airavata/data/manager/cpi/FileTransferService.java
new file mode 100644
index 0000000..676bcb1
--- /dev/null
+++ b/modules/data-manager/data-manager-cpi/src/main/java/org/apache/airavata/data/manager/cpi/FileTransferService.java
@@ -0,0 +1,212 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+*/
+package org.apache.airavata.data.manager.cpi;
+
+import org.apache.airavata.model.data.transfer.FileTransferRequestModel;
+import org.apache.airavata.model.data.transfer.LSEntryModel;
+import org.apache.airavata.model.data.transfer.StorageResourceProtocol;
+
+import java.util.List;
+
+public interface FileTransferService {
+
+    /**
+     * Method to upload the give bytes to the destination storage system
+     * @param gatewayId
+     * @param username
+     * @param fileData
+     * @param destHostname
+     * @param destLoginName
+     * @param destPort
+     * @param destProtocol
+     * @param destinationPath
+     * @param destHostCredToken
+     * @return
+     * @throws DataManagerException
+     */
+    String uploadFile(String gatewayId, String username, byte[] fileData, String destHostname, String destLoginName, int destPort,
+                    StorageResourceProtocol destProtocol, String destinationPath, String destHostCredToken)
+            throws DataManagerException;
+
+    /**
+     * Transfer file between two storage resources synchronously. Returns the file transfer request id
+     * @param gatewayId
+     * @param username
+     * @param srcHostname
+     * @param srcPort
+     * @param srcLoginName
+     * @param srcProtocol
+     * @param srcPath
+     * @param srcHostCredToken
+     * @param destHostname
+     * @param destLoginName
+     * @param destPort
+     * @param destProtocol
+     * @param destinationPath
+     * @param destHostCredToken
+     * @return
+     * @throws DataManagerException
+     */
+    String transferFile(String gatewayId, String username, String srcHostname, String srcLoginName, int srcPort,
+                      StorageResourceProtocol srcProtocol, String srcPath, String srcHostCredToken,
+                      String destHostname, String destLoginName, int destPort,
+                      StorageResourceProtocol destProtocol, String destinationPath, String destHostCredToken)
+            throws DataManagerException;
+
+    /**
+     * Transfer file between two storage resources asynchronously. Returns the file transfer request id
+     * @param gatewayId
+     * @param username
+     * @param srcHostname
+     * @param srcLoginName
+     * @param srcPort
+     * @param srcProtocol
+     * @param srcPath
+     * @param srcHostCredToken
+     * @param destHostname
+     * @param destLoginName
+     * @param destPort
+     * @param destProtocol
+     * @param destinationPath
+     * @param destHostCredToken
+     * @param callbackEmails
+     * @return
+     * @throws DataManagerException
+     */
+    String transferFileAsync(String gatewayId, String username, String srcHostname, String srcLoginName, int srcPort,
+                           StorageResourceProtocol srcProtocol, String srcPath, String srcHostCredToken,
+                           String destHostname, String destLoginName, int destPort,
+                           StorageResourceProtocol destProtocol, String destinationPath, String destHostCredToken,
+                           String[] callbackEmails)
+            throws DataManagerException;
+
+    /**
+     * Get a directory listing of the specified source directory
+     * @param hostname
+     * @param loginName
+     * @param port
+     * @param protocol
+     * @param path
+     * @param hostCredential
+     * @return
+     * @throws DataManagerException
+     */
+    List<LSEntryModel> getDirectoryListing(String hostname, String loginName, int port,
+                                       StorageResourceProtocol protocol, String path, String hostCredential)
+            throws DataManagerException;
+
+    /**
+     * Move file from one place to another inside the same storage resource
+     * @param hostname
+     * @param loginName
+     * @param port
+     * @param protocol
+     * @param hostCredential
+     * @param sourcePath
+     * @param destinationPath
+     * @throws DataManagerException
+     */
+    void moveFile(String hostname, String loginName, int port,
+                  StorageResourceProtocol protocol, String hostCredential, String sourcePath, String destinationPath)
+            throws DataManagerException;
+
+    /**
+     * Rename a file
+     * @param hostname
+     * @param loginName
+     * @param port
+     * @param protocol
+     * @param hostCredential
+     * @param sourcePath
+     * @param newName
+     * @throws DataManagerException
+     */
+    void renameFile(String hostname, String loginName, int port,
+                    StorageResourceProtocol protocol, String hostCredential, String sourcePath, String newName)
+            throws DataManagerException;
+
+    /**
+     * Create new directory
+     * @param hostname
+     * @param loginName
+     * @param port
+     * @param protocol
+     * @param hostCredential
+     * @param dirPath
+     * @throws DataManagerException
+     */
+    void mkdir(String hostname, String loginName, int port,
+               StorageResourceProtocol protocol, String hostCredential, String dirPath)
+            throws DataManagerException;
+
+    /**
+     * Delete File in storage resource
+     * @param hostname
+     * @param loginName
+     * @param port
+     * @param protocol
+     * @param hostCredential
+     * @param filePath
+     * @throws DataManagerException
+     */
+    void deleteFile(String hostname, String loginName, int port,
+                    StorageResourceProtocol protocol, String hostCredential, String filePath)
+            throws DataManagerException;
+
+    /**
+     * Check whether the specified file exists
+     * @param hostname
+     * @param loginName
+     * @param port
+     * @param protocol
+     * @param hostCredential
+     * @param filePath
+     * @return
+     * @throws DataManagerException
+     */
+    boolean isExists(String hostname, String loginName, int port,
+                    StorageResourceProtocol protocol, String hostCredential, String filePath)
+            throws DataManagerException;
+
+    /**
+     * Check whether the path points to a directory
+     * @param hostname
+     * @param loginName
+     * @param port
+     * @param protocol
+     * @param hostCredential
+     * @param filePath
+     * @return
+     * @throws DataManagerException
+     */
+    boolean isDirectory(String hostname, String loginName, int port,
+                     StorageResourceProtocol protocol, String hostCredential, String filePath)
+            throws DataManagerException;
+
+    /**
+     * Method to retrieve file transfer status giving transfer id
+     * @param transferId
+     * @return
+     * @throws DataManagerException
+     */
+    FileTransferRequestModel getFileTransferRequestStatus(String transferId)
+            throws DataManagerException;
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/airavata/blob/1ad9ae5d/modules/data-manager/data-manager-cpi/src/main/java/org/apache/airavata/data/manager/cpi/MetadataCatalogService.java
----------------------------------------------------------------------
diff --git a/modules/data-manager/data-manager-cpi/src/main/java/org/apache/airavata/data/manager/cpi/MetadataCatalogService.java b/modules/data-manager/data-manager-cpi/src/main/java/org/apache/airavata/data/manager/cpi/MetadataCatalogService.java
new file mode 100644
index 0000000..a75e0d4
--- /dev/null
+++ b/modules/data-manager/data-manager-cpi/src/main/java/org/apache/airavata/data/manager/cpi/MetadataCatalogService.java
@@ -0,0 +1,56 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+*/
+package org.apache.airavata.data.manager.cpi;
+
+import org.apache.airavata.model.data.metadata.MetadataModel;
+
+public interface MetadataCatalogService {
+
+    /**
+     * Create new metadata model
+     * @param metadataModel
+     * @return
+     * @throws DataManagerException
+     */
+    String createMetadata(MetadataModel metadataModel) throws DataManagerException;
+
+    /**
+     * Update exisiting metadata model
+     * @param metadataModel
+     * @throws DataManagerException
+     */
+    void updateMetadata(MetadataModel metadataModel) throws DataManagerException;
+
+    /**
+     * Delete existing metadata model
+     * @param metadataId
+     * @throws DataManagerException
+     */
+    void deleteMetadata(String metadataId) throws DataManagerException;
+
+    /**
+     * Retrieve metadata model
+     * @param metadataId
+     * @return
+     * @throws DataManagerException
+     */
+    MetadataModel getMetadata(String metadataId) throws DataManagerException;
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/airavata/blob/1ad9ae5d/modules/data-manager/data-manager-cpi/src/main/java/org/apache/airavata/data/manager/cpi/ReplicaCatalogService.java
----------------------------------------------------------------------
diff --git a/modules/data-manager/data-manager-cpi/src/main/java/org/apache/airavata/data/manager/cpi/ReplicaCatalogService.java b/modules/data-manager/data-manager-cpi/src/main/java/org/apache/airavata/data/manager/cpi/ReplicaCatalogService.java
new file mode 100644
index 0000000..b136c1c
--- /dev/null
+++ b/modules/data-manager/data-manager-cpi/src/main/java/org/apache/airavata/data/manager/cpi/ReplicaCatalogService.java
@@ -0,0 +1,86 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+*/
+package org.apache.airavata.data.manager.cpi;
+
+import org.apache.airavata.model.data.replica.FileCollectionModel;
+import org.apache.airavata.model.data.replica.FileModel;
+
+public interface ReplicaCatalogService {
+
+    /**
+     * Creates a new file entry in the replica catalog
+     * @param fileModel
+     * @return
+     */
+    String registerFileDetails(FileModel fileModel) throws DataManagerException;
+
+    /**
+     * Updates an existing file information
+     * @param fileModel
+     */
+    void updateFileDetails(FileModel fileModel) throws DataManagerException;
+
+
+    /**
+     * Deletes the specified file details entry
+     * @param fileId
+     */
+    void deleteFileDetails(String fileId) throws DataManagerException;
+
+
+    /**
+     * Retrieves file details for the specified file id
+     * @param fileId
+     * @return
+     */
+    FileModel getFileDetails(String fileId) throws DataManagerException;
+
+
+    /**
+     * Create new file collection entry
+     * @param fileCollectionModel
+     * @return
+     * @throws DataManagerException
+     */
+    String registerFileCollection(FileCollectionModel fileCollectionModel) throws DataManagerException;
+
+    /**
+     * Update existing file collection
+     * @param fileCollectionModel
+     * @throws DataManagerException
+     */
+    void updateFileCollection(FileCollectionModel fileCollectionModel) throws DataManagerException;
+
+    /**
+     * Delete exisiting file collection
+     * @param collectionId
+     * @throws DataManagerException
+     */
+    void deleteFileCollection(String collectionId) throws DataManagerException;
+
+    /**
+     * Retrieve file collection specifying the collection id
+     * @param collectionId
+     * @return
+     * @throws DataManagerException
+     */
+    FileCollectionModel getFileCollection(String collectionId) throws DataManagerException;
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/airavata/blob/1ad9ae5d/modules/data-manager/pom.xml
----------------------------------------------------------------------
diff --git a/modules/data-manager/pom.xml b/modules/data-manager/pom.xml
new file mode 100644
index 0000000..eb81d40
--- /dev/null
+++ b/modules/data-manager/pom.xml
@@ -0,0 +1,54 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <parent>
+        <artifactId>airavata</artifactId>
+        <groupId>org.apache.airavata</groupId>
+        <version>0.16-SNAPSHOT</version>
+        <relativePath>../../pom.xml</relativePath>
+    </parent>
+
+    <modelVersion>4.0.0</modelVersion>
+    <artifactId>data-manager</artifactId>
+    <packaging>pom</packaging>
+    <name>Airavata Data Manager</name>
+    <url>http://airavata.apache.org/</url>
+
+    <modules>
+        <module>data-manager-cpi</module>
+        <module>data-manager-core</module>
+    </modules>
+    <dependencies>
+        <dependency>
+            <groupId>org.apache.airavata</groupId>
+            <artifactId>airavata-data-models</artifactId>
+            <version>${project.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.airavata</groupId>
+            <artifactId>airavata-registry-core</artifactId>
+            <version>${project.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.airavata</groupId>
+            <artifactId>airavata-registry-cpi</artifactId>
+            <version>${project.version}</version>
+        </dependency>
+
+        <dependency>
+            <groupId>org.slf4j</groupId>
+            <artifactId>slf4j-api</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>log4j</groupId>
+            <artifactId>log4j</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>junit</groupId>
+            <artifactId>junit</artifactId>
+            <version>4.7</version>
+            <scope>test</scope>
+        </dependency>
+    </dependencies>
+</project>
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/airavata/blob/1ad9ae5d/modules/file-manager/file-manager-core/pom.xml
----------------------------------------------------------------------
diff --git a/modules/file-manager/file-manager-core/pom.xml b/modules/file-manager/file-manager-core/pom.xml
deleted file mode 100644
index b8e98e2..0000000
--- a/modules/file-manager/file-manager-core/pom.xml
+++ /dev/null
@@ -1,95 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<project xmlns="http://maven.apache.org/POM/4.0.0"
-         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
-    <parent>
-        <artifactId>file-manager</artifactId>
-        <groupId>org.apache.airavata</groupId>
-        <version>0.16-SNAPSHOT</version>
-        <relativePath>../pom.xml</relativePath>
-    </parent>
-
-    <modelVersion>4.0.0</modelVersion>
-    <artifactId>file-manager-core</artifactId>
-    <packaging>jar</packaging>
-    <name>Airavata File Manager Core</name>
-    <url>http://airavata.apache.org/</url>
-
-    <dependencies>
-        <dependency>
-            <groupId>org.apache.airavata</groupId>
-            <artifactId>airavata-data-models</artifactId>
-            <version>${project.version}</version>
-        </dependency>
-        <dependency>
-            <groupId>org.apache.airavata</groupId>
-            <artifactId>airavata-registry-core</artifactId>
-            <version>${project.version}</version>
-        </dependency>
-        <dependency>
-            <groupId>org.apache.airavata</groupId>
-            <artifactId>airavata-registry-cpi</artifactId>
-            <version>${project.version}</version>
-        </dependency>
-        <dependency>
-            <groupId>org.apache.airavata</groupId>
-            <artifactId>airavata-commons</artifactId>
-            <version>${project.version}</version>
-        </dependency>
-        <dependency>
-            <groupId>org.apache.airavata</groupId>
-            <artifactId>airavata-server-configuration</artifactId>
-            <version>${project.version}</version>
-            <scope>test</scope>
-        </dependency>
-        <dependency>
-            <groupId>org.apache.airavata</groupId>
-            <artifactId>file-manager-cpi</artifactId>
-            <version>${project.version}</version>
-        </dependency>
-        <dependency>
-            <groupId>org.apache.airavata</groupId>
-            <artifactId>airavata-registry-core</artifactId>
-            <version>${project.version}</version>
-        </dependency>
-        <dependency>
-            <groupId>org.apache.airavata</groupId>
-            <artifactId>airavata-registry-cpi</artifactId>
-            <version>${project.version}</version>
-        </dependency>
-
-        <dependency>
-            <groupId>org.jglobus</groupId>
-            <artifactId>gss</artifactId>
-            <version>${jglobus.version}</version>
-        </dependency>
-        <dependency>
-            <groupId>org.jglobus</groupId>
-            <artifactId>gridftp</artifactId>
-            <version>${jglobus.version}</version>
-        </dependency>
-        <dependency>
-            <groupId>org.jglobus</groupId>
-            <artifactId>myproxy</artifactId>
-            <version>2.1.0</version>
-        </dependency>
-
-        <dependency>
-            <groupId>org.bouncycastle</groupId>
-            <artifactId>bcprov-jdk16</artifactId>
-            <version>1.44</version>
-        </dependency>
-
-
-        <dependency>
-            <groupId>com.fasterxml.jackson.core</groupId>
-            <artifactId>jackson-databind</artifactId>
-            <version>2.5.3</version>
-        </dependency>
-        <dependency>
-            <groupId>org.mongodb</groupId>
-            <artifactId>mongo-java-driver</artifactId>
-            <version>3.0.0</version>
-        </dependency>
-    </dependencies>
-</project>
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/airavata/blob/1ad9ae5d/modules/file-manager/file-manager-core/src/main/java/org/apache/airavata/file/manager/core/FileManagerFactory.java
----------------------------------------------------------------------
diff --git a/modules/file-manager/file-manager-core/src/main/java/org/apache/airavata/file/manager/core/FileManagerFactory.java b/modules/file-manager/file-manager-core/src/main/java/org/apache/airavata/file/manager/core/FileManagerFactory.java
deleted file mode 100644
index 52c76cb..0000000
--- a/modules/file-manager/file-manager-core/src/main/java/org/apache/airavata/file/manager/core/FileManagerFactory.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
-*/
-package org.apache.airavata.file.manager.core;
-
-import org.apache.airavata.file.manager.cpi.FileManager;
-import org.apache.airavata.file.manager.cpi.FileManagerException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class FileManagerFactory {
-    private final static Logger logger = LoggerFactory.getLogger(FileManagerFactory.class);
-
-    public static FileManager getFileManager() throws FileManagerException {
-        return new FileManagerImpl();
-    }
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/airavata/blob/1ad9ae5d/modules/file-manager/file-manager-core/src/main/java/org/apache/airavata/file/manager/core/FileManagerImpl.java
----------------------------------------------------------------------
diff --git a/modules/file-manager/file-manager-core/src/main/java/org/apache/airavata/file/manager/core/FileManagerImpl.java b/modules/file-manager/file-manager-core/src/main/java/org/apache/airavata/file/manager/core/FileManagerImpl.java
deleted file mode 100644
index dee470a..0000000
--- a/modules/file-manager/file-manager-core/src/main/java/org/apache/airavata/file/manager/core/FileManagerImpl.java
+++ /dev/null
@@ -1,75 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
-*/
-package org.apache.airavata.file.manager.core;
-
-import org.apache.airavata.file.manager.cpi.*;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-
-public class FileManagerImpl implements FileManager {
-    private final static Logger logger = LoggerFactory.getLogger(FileManagerImpl.class);
-
-    /**
-     * Return file transfer service instance
-     *
-     * @return
-     */
-    @Override
-    public FileTransferService getFileTransferService()  throws FileManagerException {
-        try{
-            return new FileTransferServiceImpl();
-        }catch (Exception e){
-            logger.error(e.getMessage(), e);
-            throw new FileManagerException(e);
-        }
-    }
-
-    /**
-     * Return replica catalog service instance
-     *
-     * @return
-     */
-    @Override
-    public ReplicaCatalogService getReplicaCatalogService()  throws FileManagerException{
-        try{
-            return new ReplicaCatalogServiceImpl();
-        }catch (Exception e){
-            logger.error(e.getMessage(), e);
-            throw new FileManagerException(e);
-        }
-    }
-
-    /**
-     * Return metadata catalog service
-     *
-     * @return
-     */
-    @Override
-    public MetadataCatalogService getMetadataCatalogService()  throws FileManagerException{
-        try{
-            return new MetadataCatalogServiceImpl();
-        }catch (Exception e){
-            logger.error(e.getMessage(), e);
-            throw new FileManagerException(e);
-        }
-    }
-}
\ No newline at end of file