You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cloudstack.apache.org by GitBox <gi...@apache.org> on 2018/01/18 15:43:03 UTC
[GitHub] pdube closed pull request #1830: CLOUDSTACK-9677: Adding storage policy support for swift as secondary?
pdube closed pull request #1830: CLOUDSTACK-9677: Adding storage policy support for swift as secondary?
URL: https://github.com/apache/cloudstack/pull/1830
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/api/src/com/cloud/agent/api/to/SwiftTO.java b/api/src/com/cloud/agent/api/to/SwiftTO.java
index c7a98660302..b89dfea40e0 100644
--- a/api/src/com/cloud/agent/api/to/SwiftTO.java
+++ b/api/src/com/cloud/agent/api/to/SwiftTO.java
@@ -26,17 +26,19 @@
String userName;
String key;
+ String storagePolicy;
private static final String pathSeparator = "/";
public SwiftTO() {
}
- public SwiftTO(Long id, String url, String account, String userName, String key) {
+ public SwiftTO(Long id, String url, String account, String userName, String key, String storagePolicy) {
this.id = id;
this.url = url;
this.account = account;
this.userName = userName;
this.key = key;
+ this.storagePolicy = storagePolicy;
}
public Long getId() {
@@ -63,6 +65,11 @@ public String getKey() {
return key;
}
+ @Override
+ public String getStoragePolicy() {
+ return this.storagePolicy;
+ }
+
@Override
public DataStoreRole getRole() {
return DataStoreRole.Image;
diff --git a/api/src/org/apache/cloudstack/api/ApiConstants.java b/api/src/org/apache/cloudstack/api/ApiConstants.java
index 00e9d388c28..5f83f1f6db7 100644
--- a/api/src/org/apache/cloudstack/api/ApiConstants.java
+++ b/api/src/org/apache/cloudstack/api/ApiConstants.java
@@ -252,6 +252,7 @@
public static final String STATE = "state";
public static final String STATUS = "status";
public static final String STORAGE_TYPE = "storagetype";
+ public static final String STORAGE_POLICY = "storagepolicy";
public static final String STORAGE_MOTION_ENABLED = "storagemotionenabled";
public static final String STORAGE_CAPABILITIES = "storagecapabilities";
public static final String SYSTEM_VM_TYPE = "systemvmtype";
diff --git a/plugins/hypervisors/xenserver/src/com/cloud/hypervisor/xenserver/resource/XenServerStorageProcessor.java b/plugins/hypervisors/xenserver/src/com/cloud/hypervisor/xenserver/resource/XenServerStorageProcessor.java
index 4999fd12b86..c5b1cb8c05a 100644
--- a/plugins/hypervisors/xenserver/src/com/cloud/hypervisor/xenserver/resource/XenServerStorageProcessor.java
+++ b/plugins/hypervisors/xenserver/src/com/cloud/hypervisor/xenserver/resource/XenServerStorageProcessor.java
@@ -78,6 +78,7 @@
import java.io.File;
import java.net.URI;
+import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
@@ -1063,21 +1064,55 @@ public Answer copyVolumeFromPrimaryToSecondary(final CopyCommand cmd) {
}
boolean swiftUpload(final Connection conn, final SwiftTO swift, final String container, final String ldir, final String lfilename, final Boolean isISCSI, final int wait) {
- String result = null;
+
+ List<String> params = getSwiftParams(swift, container, ldir, lfilename, isISCSI);
+
try {
- result =
- hypervisorResource.callHostPluginAsync(conn, "swiftxenserver", "swift", wait, "op", "upload", "url", swift.getUrl(), "account", swift.getAccount(), "username",
- swift.getUserName(), "key", swift.getKey(), "container", container, "ldir", ldir, "lfilename", lfilename, "isISCSI", isISCSI.toString());
- if (result != null && result.equals("true")) {
- return true;
- }
+ String result = hypervisorResource.callHostPluginAsync(conn, "swiftxenserver", "swift", wait, params.toArray(new String[params.size()]));
+ return "true".equals(result);
} catch (final Exception e) {
s_logger.warn("swift upload failed due to " + e.toString(), e);
}
return false;
}
- protected String deleteSnapshotBackup(final Connection conn, final String localMountPoint, final String path, final String secondaryStorageMountPath, final String backupUUID) {
+ private List<String> getSwiftParams(SwiftTO swift, String container, String ldir, String lfilename,
+ Boolean isISCSI) {
+
+ // ORDER IS IMPORTANT
+ List<String> params = new ArrayList<>();
+ //operation
+ params.add("op");
+ params.add("upload");
+
+ //auth
+ params.add("url");
+ params.add(swift.getUrl());
+ params.add("account");
+ params.add(swift.getAccount());
+ params.add("username");
+ params.add(swift.getUserName());
+ params.add("key");
+ params.add(swift.getKey());
+
+ // object info
+ params.add("container");
+ params.add(container);
+ params.add("ldir");
+ params.add(ldir);
+ params.add("lfilename");
+ params.add(lfilename);
+ params.add("isISCSI");
+ params.add(isISCSI.toString());
+
+ if (swift.getStoragePolicy() != null) {
+ params.add("storagepolicy");
+ params.add(swift.getStoragePolicy());
+ }
+ return params;
+ }
+
+ protected String deleteSnapshotBackup(final Connection conn, final String localMountPoint, final String path, final String secondaryStorageMountPath, final String backupUUID) {
// If anybody modifies the formatting below again, I'll skin them
final String result =
diff --git a/plugins/storage/image/swift/src/org/apache/cloudstack/storage/datastore/driver/SwiftImageStoreDriverImpl.java b/plugins/storage/image/swift/src/org/apache/cloudstack/storage/datastore/driver/SwiftImageStoreDriverImpl.java
index 2816c6063f6..7e1486214bc 100644
--- a/plugins/storage/image/swift/src/org/apache/cloudstack/storage/datastore/driver/SwiftImageStoreDriverImpl.java
+++ b/plugins/storage/image/swift/src/org/apache/cloudstack/storage/datastore/driver/SwiftImageStoreDriverImpl.java
@@ -68,7 +68,7 @@
public DataStoreTO getStoreTO(DataStore store) {
ImageStoreImpl imgStore = (ImageStoreImpl)store;
Map<String, String> details = _imageStoreDetailsDao.getDetails(imgStore.getId());
- return new SwiftTO(imgStore.getId(), imgStore.getUri(), details.get(ApiConstants.ACCOUNT), details.get(ApiConstants.USERNAME), details.get(ApiConstants.KEY));
+ return new SwiftTO(imgStore.getId(), imgStore.getUri(), details.get(ApiConstants.ACCOUNT), details.get(ApiConstants.USERNAME), details.get(ApiConstants.KEY), details.get(ApiConstants.STORAGE_POLICY));
}
@Override
diff --git a/scripts/storage/secondary/swift b/scripts/storage/secondary/swift
index 4138db8b17e..1b2cfb7e8a9 100755
--- a/scripts/storage/secondary/swift
+++ b/scripts/storage/secondary/swift
@@ -1473,8 +1473,9 @@ post [options] [container] [object]
Updates meta information for the account, container, or object depending on
the args given. If the container is not found, it will be created
automatically; but this is not true for accounts and objects. Containers
- also allow the -r (or --read-acl) and -w (or --write-acl) options. The -m
- or --meta option is allowed on all and used to define the user meta data
+ also allow the -r (or --read-acl) and -w (or --write-acl) options.
+ The --storage-policy will set a storage policy to the container if the container does not exist.
+ The -m or --meta option is allowed on all and used to define the user meta data
items to set in the form Name:Value. This option can be repeated. Example:
post -m Color:Blue -m Size:Large'''.strip('\n')
@@ -1493,6 +1494,8 @@ def st_post(options, args, print_queue, error_queue):
parser.add_option('-m', '--meta', action='append', dest='meta', default=[],
help='Sets a meta data item with the syntax name:value. This option '
'may be repeated. Example: -m Color:Blue -m Size:Large')
+ parser.add_option('', '--storage-policy', action='store', dest='storage_policy',
+ help='Sets a storage policy to the container if the container does not exist')
(options, args) = parse_args(parser, args)
args = args[1:]
if (options.read_acl or options.write_acl or options.sync_to or
@@ -1529,6 +1532,8 @@ def st_post(options, args, print_queue, error_queue):
headers['X-Container-Sync-To'] = options.sync_to
if options.sync_key is not None:
headers['X-Container-Sync-Key'] = options.sync_key
+ if options.storage_policy is not None:
+ headers['X-Storage-Policy'] = options.storage_policy
try:
conn.post_container(args[0], headers=headers)
except ClientException, err:
@@ -1558,7 +1563,8 @@ upload [options] container file_or_directory [file_or_directory] [...]
Uploads to the given container the files and directories specified by the
remaining args. -c or --changed is an option that will only upload files
that have changed since the last upload. -S <size> or --segment-size <size>
- and --leave-segments are options as well (see --help for more).
+ and --leave-segments are options as well (see --help for more). --storage-policy
+ Sets a storage policy to the container if the container does not exist.
'''.strip('\n')
@@ -1576,6 +1582,8 @@ def st_upload(options, args, print_queue, error_queue):
dest='leave_segments', default=False, help='Indicates that you want '
'the older segments of manifest objects left alone (in the case of '
'overwrites)')
+ parser.add_option('', '--storage-policy', action='store', dest='storage_policy',
+ help='Sets a storage policy to the container if the container does not exist')
(options, args) = parse_args(parser, args)
args = args[1:]
if len(args) < 2:
@@ -1749,9 +1757,12 @@ def st_upload(options, args, print_queue, error_queue):
# permissions, so we'll ignore any error. If there's really a problem,
# it'll surface on the first object PUT.
try:
- conn.put_container(args[0])
+ container_headers = {}
+ if options.storage_policy is not None:
+ container_headers['X-Storage-Policy'] = options.storage_policy
+ conn.put_container(args[0],headers=container_headers)
if options.segment_size is not None:
- conn.put_container(args[0] + '_segments')
+ conn.put_container(args[0] + '_segments',headers=container_headers)
except Exception:
pass
try:
diff --git a/scripts/vm/hypervisor/xenserver/swift b/scripts/vm/hypervisor/xenserver/swift
index c9d2cebb04f..3bb6b2dd8ce 100755
--- a/scripts/vm/hypervisor/xenserver/swift
+++ b/scripts/vm/hypervisor/xenserver/swift
@@ -1475,8 +1475,9 @@ post [options] [container] [object]
Updates meta information for the account, container, or object depending on
the args given. If the container is not found, it will be created
automatically; but this is not true for accounts and objects. Containers
- also allow the -r (or --read-acl) and -w (or --write-acl) options. The -m
- or --meta option is allowed on all and used to define the user meta data
+ also allow the -r (or --read-acl) and -w (or --write-acl) options.
+ The --storage-policy will set a storage policy to the container if the container does not exist.
+ The -m or --meta option is allowed on all and used to define the user meta data
items to set in the form Name:Value. This option can be repeated. Example:
post -m Color:Blue -m Size:Large'''.strip('\n')
@@ -1495,6 +1496,8 @@ def st_post(options, args, print_queue, error_queue):
parser.add_option('-m', '--meta', action='append', dest='meta', default=[],
help='Sets a meta data item with the syntax name:value. This option '
'may be repeated. Example: -m Color:Blue -m Size:Large')
+ parser.add_option('', '--storage-policy', action='store', dest='storage_policy',
+ help='Sets a storage policy to the container if the container does not exist')
(options, args) = parse_args(parser, args)
args = args[1:]
if (options.read_acl or options.write_acl or options.sync_to or
@@ -1531,6 +1534,8 @@ def st_post(options, args, print_queue, error_queue):
headers['X-Container-Sync-To'] = options.sync_to
if options.sync_key is not None:
headers['X-Container-Sync-Key'] = options.sync_key
+ if options.storage_policy is not None:
+ headers['X-Storage-Policy'] = options.storage_policy
try:
conn.post_container(args[0], headers=headers)
except ClientException, err:
@@ -1560,7 +1565,8 @@ upload [options] container file_or_directory [file_or_directory] [...]
Uploads to the given container the files and directories specified by the
remaining args. -c or --changed is an option that will only upload files
that have changed since the last upload. -S <size> or --segment-size <size>
- and --leave-segments are options as well (see --help for more).
+ and --leave-segments are options as well (see --help for more). --storage-policy
+ Sets a storage policy to the container if the container does not exist.
'''.strip('\n')
@@ -1578,6 +1584,8 @@ def st_upload(options, args, print_queue, error_queue):
dest='leave_segments', default=False, help='Indicates that you want '
'the older segments of manifest objects left alone (in the case of '
'overwrites)')
+ parser.add_option('', '--storage-policy', action='store', dest='storage_policy',
+ help='Sets a storage policy to the container if the container does not exist')
(options, args) = parse_args(parser, args)
args = args[1:]
if len(args) < 2:
@@ -1751,9 +1759,12 @@ def st_upload(options, args, print_queue, error_queue):
# permissions, so we'll ignore any error. If there's really a problem,
# it'll surface on the first object PUT.
try:
- conn.put_container(args[0])
+ container_headers = {}
+ if options.storage_policy is not None:
+ container_headers['X-Storage-Policy'] = options.storage_policy
+ conn.put_container(args[0],headers=container_headers)
if options.segment_size is not None:
- conn.put_container(args[0] + '_segments')
+ conn.put_container(args[0] + '_segments',headers=container_headers)
except Exception:
pass
try:
diff --git a/scripts/vm/hypervisor/xenserver/swiftxenserver b/scripts/vm/hypervisor/xenserver/swiftxenserver
index b0be24f5efe..756d91d1df8 100644
--- a/scripts/vm/hypervisor/xenserver/swiftxenserver
+++ b/scripts/vm/hypervisor/xenserver/swiftxenserver
@@ -52,6 +52,9 @@ def upload(args):
lfilename = args['lfilename']
isISCSI = args['isISCSI']
segment = 0
+ storagepolicy = None
+ if "storagepolicy" in args:
+ storagepolicy = args["storagepolicy"]
logging.debug("#### VMOPS upload %s to swift ####", lfilename)
savedpath = os.getcwd()
os.chdir(ldir)
@@ -72,6 +75,9 @@ def upload(args):
cmd = [SWIFT, "-A", url, "-U", account + ":" + username, "-K", key, "upload", "-S", str(MAX_SEG_SIZE), container, lfilename]
else :
cmd = [SWIFT, "-A", url ,"-U", account + ":" + username, "-K", key, "upload", container, lfilename]
+ if storagepolicy is not None:
+ cmd.append("--storage-policy")
+ cmd.append(storagepolicy)
util.pread2(cmd)
return 'true'
finally:
diff --git a/ui/l10n/en.js b/ui/l10n/en.js
index 075a0d801f1..116bafa87c3 100644
--- a/ui/l10n/en.js
+++ b/ui/l10n/en.js
@@ -1566,6 +1566,7 @@ var dictionary = {"ICMP.code":"ICMP Code",
"label.storage.tags":"Storage Tags",
"label.storage.traffic":"Storage Traffic",
"label.storage.type":"Storage Type",
+"label.storagepolicy":"Storage policy",
"label.subdomain.access":"Subdomain Access",
"label.submit":"Submit",
"label.submitted.by":"[Submitted by: <span id=\"submitted_by\"></span>]",
diff --git a/ui/scripts/system.js b/ui/scripts/system.js
index 27825f9df43..13eb51f57ca 100644
--- a/ui/scripts/system.js
+++ b/ui/scripts/system.js
@@ -19404,6 +19404,7 @@
$form.find('.form-item[rel=account]').hide();
$form.find('.form-item[rel=username]').hide();
$form.find('.form-item[rel=key]').hide();
+ $form.find('.form-item[rel=storagepolicy]').hide();
} else if ($(this).val() == "SMB") {
//NFS, SMB
$form.find('.form-item[rel=zoneid]').css('display', 'inline-block');
@@ -19436,6 +19437,7 @@
$form.find('.form-item[rel=account]').hide();
$form.find('.form-item[rel=username]').hide();
$form.find('.form-item[rel=key]').hide();
+ $form.find('.form-item[rel=storagepolicy]').hide();
} else if ($(this).val() == "S3") {
//NFS, SMB
$form.find('.form-item[rel=zoneid]').hide();
@@ -19470,6 +19472,7 @@
$form.find('.form-item[rel=account]').hide();
$form.find('.form-item[rel=username]').hide();
$form.find('.form-item[rel=key]').hide();
+ $form.find('.form-item[rel=storagepolicy]').hide();
} else if ($(this).val() == "Swift") {
//NFS, SMB
$form.find('.form-item[rel=zoneid]').hide();
@@ -19502,6 +19505,7 @@
$form.find('.form-item[rel=account]').css('display', 'inline-block');
$form.find('.form-item[rel=username]').css('display', 'inline-block');
$form.find('.form-item[rel=key]').css('display', 'inline-block');
+ $form.find('.form-item[rel=storagepolicy]').css('display', 'inline-block');
}
});
@@ -19692,14 +19696,26 @@
}
},
account: {
- label: 'label.account'
+ label: 'label.account',
+ validation: {
+ required: true
+ }
},
username: {
- label: 'label.username'
+ label: 'label.username',
+ validation: {
+ required: true
+ }
},
key: {
- label: 'label.key'
- }
+ label: 'label.key',
+ validation: {
+ required: true
+ }
+ },
+ storagepolicy: {
+ label: 'label.storagepolicy'
+ }
//Swift (end)
}
},
@@ -19866,6 +19882,11 @@
data[ 'details[' + index.toString() + '].value'] = args.data.key;
index++;
}
+ if (args.data.storagepolicy != null && args.data.storagepolicy.length > 0) {
+ data[ 'details[' + index.toString() + '].key'] = 'storagepolicy';
+ data[ 'details[' + index.toString() + '].value'] = args.data.storagepolicy;
+ index++;
+ }
$.ajax({
url: createURL('addImageStore'),
data: data,
diff --git a/utils/src/main/java/com/cloud/utils/SwiftUtil.java b/utils/src/main/java/com/cloud/utils/SwiftUtil.java
index ce1bee36b62..685a5dde5c7 100644
--- a/utils/src/main/java/com/cloud/utils/SwiftUtil.java
+++ b/utils/src/main/java/com/cloud/utils/SwiftUtil.java
@@ -40,10 +40,13 @@
public class SwiftUtil {
private static Logger logger = Logger.getLogger(SwiftUtil.class);
- private static final long SWIFT_MAX_SIZE = 5L * 1024L * 1024L * 1024L;
+ protected static final long SWIFT_MAX_SIZE = 5L * 1024L * 1024L * 1024L;
private static final String HMAC_SHA1_ALGORITHM = "HmacSHA1";
-
-
+ private static final String CD_SRC = "cd %s;";
+ private static final String SWIFT_CMD= "/usr/bin/python %s -A %s -U %s:%s -K %s %s";
+ private static final String WITH_STORAGE_POLICY = " --storage-policy \"%s\"";
+ private static final String WITH_SEGMENTS = " -S "+SWIFT_MAX_SIZE;
+ private static final String[] OPERATIONS_WITH_STORAGE_POLICIES = {"post","upload"};
public interface SwiftClientCfg {
String getAccount();
@@ -53,6 +56,8 @@
String getKey();
String getEndPoint();
+
+ String getStoragePolicy();
}
private static String getSwiftCLIPath() {
@@ -65,19 +70,10 @@ private static String getSwiftCLIPath() {
}
public static boolean postMeta(SwiftClientCfg cfg, String container, String object, Map<String, String> metas) {
- String swiftCli = getSwiftCLIPath();
- StringBuilder cms = new StringBuilder();
- for (Map.Entry<String, String> entry : metas.entrySet()) {
- cms.append(" -m ");
- cms.append(entry.getKey());
- cms.append(":");
- cms.append(entry.getValue());
- cms.append(" ");
- }
Script command = new Script("/bin/bash", logger);
command.add("-c");
- command.add("/usr/bin/python " + swiftCli + " -A " + cfg.getEndPoint() + " -U " + cfg.getAccount() + ":" + cfg.getUserName() + " -K " + cfg.getKey() + " post " +
- container + " " + object + " " + cms.toString());
+ command.add(getSwiftObjectCmd(cfg, getSwiftCLIPath(),"post", container, object) + getMeta(metas));
+
OutputInterpreter.OneLineParser parser = new OutputInterpreter.OneLineParser();
String result = command.execute(parser);
if (result != null) {
@@ -87,21 +83,14 @@ public static boolean postMeta(SwiftClientCfg cfg, String container, String obje
}
public static String putObject(SwiftClientCfg cfg, File srcFile, String container, String fileName) {
- String swiftCli = getSwiftCLIPath();
if (fileName == null) {
fileName = srcFile.getName();
}
- String srcDirectory = srcFile.getParent();
+
Script command = new Script("/bin/bash", logger);
- long size = srcFile.length();
command.add("-c");
- if (size <= SWIFT_MAX_SIZE) {
- command.add("cd " + srcDirectory + ";/usr/bin/python " + swiftCli + " -A " + cfg.getEndPoint() + " -U " + cfg.getAccount() + ":" + cfg.getUserName() +
- " -K " + cfg.getKey() + " upload " + container + " " + fileName);
- } else {
- command.add("cd " + srcDirectory + ";/usr/bin/python " + swiftCli + " -A " + cfg.getEndPoint() + " -U " + cfg.getAccount() + ":" + cfg.getUserName() +
- " -K " + cfg.getKey() + " upload -S " + SWIFT_MAX_SIZE + " " + container + " " + fileName);
- }
+ command.add(String.format(CD_SRC, srcFile.getParent())+getUploadObjectCommand(cfg, getSwiftCLIPath(), container,fileName, srcFile.length()));
+
OutputInterpreter.AllLinesParser parser = new OutputInterpreter.AllLinesParser();
String result = command.execute(parser);
if (result != null) {
@@ -120,38 +109,19 @@ public static String putObject(SwiftClientCfg cfg, File srcFile, String containe
return container + File.separator + srcFile.getName();
}
- private static StringBuilder buildSwiftCmd(SwiftClientCfg swift) {
- String swiftCli = getSwiftCLIPath();
- StringBuilder sb = new StringBuilder();
- sb.append(" /usr/bin/python ");
- sb.append(swiftCli);
- sb.append(" -A ");
- sb.append(swift.getEndPoint());
- sb.append(" -U ");
- sb.append(swift.getAccount());
- sb.append(":");
- sb.append(swift.getUserName());
- sb.append(" -K ");
- sb.append(swift.getKey());
- sb.append(" ");
- return sb;
- }
-
public static String[] list(SwiftClientCfg swift, String container, String rFilename) {
- getSwiftCLIPath();
- Script command = new Script("/bin/bash", logger);
- command.add("-c");
-
- StringBuilder swiftCmdBuilder = buildSwiftCmd(swift);
- swiftCmdBuilder.append(" list ");
- swiftCmdBuilder.append(container);
+ StringBuilder swiftCmdBuilder = new StringBuilder();
+ swiftCmdBuilder.append(getSwiftContainerCmd(swift, getSwiftCLIPath(), "list", container));
if (rFilename != null) {
swiftCmdBuilder.append(" -p ");
swiftCmdBuilder.append(rFilename);
}
+ Script command = new Script("/bin/bash", logger);
+ command.add("-c");
command.add(swiftCmdBuilder.toString());
+
OutputInterpreter.AllLinesParser parser = new OutputInterpreter.AllLinesParser();
String result = command.execute(parser);
if (result == null && parser.getLines() != null && !parser.getLines().equalsIgnoreCase("")) {
@@ -178,11 +148,11 @@ public static File getObject(SwiftClientCfg cfg, File destDirectory, String swif
} else {
destFilePath = destDirectory.getAbsolutePath();
}
- String swiftCli = getSwiftCLIPath();
+
Script command = new Script("/bin/bash", logger);
command.add("-c");
- command.add("/usr/bin/python " + swiftCli + " -A " + cfg.getEndPoint() + " -U " + cfg.getAccount() + ":" + cfg.getUserName() + " -K " + cfg.getKey() +
- " download " + container + " " + srcPath + " -o " + destFilePath);
+ command.add(getSwiftObjectCmd(cfg, getSwiftCLIPath(), "download", container, srcPath)+" -o " + destFilePath);
+
OutputInterpreter.AllLinesParser parser = new OutputInterpreter.AllLinesParser();
String result = command.execute(parser);
if (result != null) {
@@ -203,27 +173,6 @@ public static File getObject(SwiftClientCfg cfg, File destDirectory, String swif
return new File(destFilePath);
}
- public static String getContainerName(String type, Long id) {
- if (type.startsWith("T")) {
- return "T-" + id;
- } else if (type.startsWith("S")) {
- return "S-" + id;
- } else if (type.startsWith("V")) {
- return "V-" + id;
- }
- return null;
- }
-
- public static String[] splitSwiftPath(String path) {
- int index = path.indexOf(File.separator);
- if (index == -1) {
- return null;
- }
- String[] paths = new String[2];
- paths[0] = path.substring(0, index);
- paths[1] = path.substring(index + 1);
- return paths;
- }
public static boolean deleteObject(SwiftClientCfg cfg, String path) {
Script command = new Script("/bin/bash", logger);
@@ -236,13 +185,8 @@ public static boolean deleteObject(SwiftClientCfg cfg, String path) {
String container = paths[0];
String objectName = paths[1];
- StringBuilder swiftCmdBuilder = buildSwiftCmd(cfg);
- swiftCmdBuilder.append(" delete ");
- swiftCmdBuilder.append(container);
- swiftCmdBuilder.append(" ");
- swiftCmdBuilder.append(objectName);
+ command.add(getSwiftObjectCmd(cfg, getSwiftCLIPath(), "delete", container, objectName));
- command.add(swiftCmdBuilder.toString());
OutputInterpreter.AllLinesParser parser = new OutputInterpreter.AllLinesParser();
command.execute(parser);
return true;
@@ -284,7 +228,7 @@ public static URL generateTempUrl(SwiftClientCfg cfg, String container, String o
}
- public static String calculateRFC2104HMAC(String data, String key)
+ static String calculateRFC2104HMAC(String data, String key)
throws SignatureException, NoSuchAlgorithmException, InvalidKeyException {
SecretKeySpec signingKey = new SecretKeySpec(key.getBytes(), HMAC_SHA1_ALGORITHM);
@@ -294,12 +238,80 @@ public static String calculateRFC2104HMAC(String data, String key)
}
- public static String toHexString(byte[] bytes) {
+ static String toHexString(byte[] bytes) {
+ try(Formatter formatter = new Formatter()){
+ for (byte b : bytes) {
+ formatter.format("%02x", b);
+ }
+ return formatter.toString();
+ }
+ }
+
+ /////////////// SWIFT CMD STRING HELPERS ///////////////
+ protected static String getSwiftCmd(SwiftClientCfg cfg, String swiftCli, String operation){
+ return String.format(SWIFT_CMD, swiftCli,cfg.getEndPoint(),cfg.getAccount(),cfg.getUserName(),cfg.getKey(),operation);
+ }
+
+ protected static String getSwiftObjectCmd(SwiftClientCfg cfg, String swiftCliPath, String operation,String container, String objectName) {
+ String cmd = getSwiftCmd(cfg,swiftCliPath, operation) +" "+ container+" "+objectName;
+ if(StringUtils.isNotBlank(cfg.getStoragePolicy()) && supportsStoragePolicies(operation)){
+ return cmd + String.format(WITH_STORAGE_POLICY, cfg.getStoragePolicy());
+ }
+ return cmd;
+ }
+
+ private static boolean supportsStoragePolicies(String operation) {
+ for(String supportedOp: OPERATIONS_WITH_STORAGE_POLICIES){
+ if(supportedOp.equals(operation)){
+ return true;
+ }
+ }
+ return false;
+ }
+
+ protected static String getSwiftContainerCmd(SwiftClientCfg cfg, String swiftCliPath, String operation, String container) {
+ return getSwiftCmd(cfg,swiftCliPath, operation) +" "+ container;
+ }
+
+ protected static String getUploadObjectCommand(SwiftClientCfg cfg, String swiftCliPath, String container, String objectName, long size) {
+ String cmd = getSwiftObjectCmd(cfg, swiftCliPath, "upload", container, objectName);
+ if(size > SWIFT_MAX_SIZE){
+ return cmd + WITH_SEGMENTS;
+ }
+ return cmd;
+ }
- Formatter formatter = new Formatter();
- for (byte b : bytes) {
- formatter.format("%02x", b);
+ public static String getContainerName(String type, Long id) {
+ if (type.startsWith("T")) {
+ return "T-" + id;
+ } else if (type.startsWith("S")) {
+ return "S-" + id;
+ } else if (type.startsWith("V")) {
+ return "V-" + id;
+ }
+ return null;
+ }
+
+ public static String[] splitSwiftPath(String path) {
+ int index = path.indexOf(File.separator);
+ if (index == -1) {
+ return null;
+ }
+ String[] paths = new String[2];
+ paths[0] = path.substring(0, index);
+ paths[1] = path.substring(index + 1);
+ return paths;
+ }
+
+ private static String getMeta(Map<String, String> metas) {
+ StringBuilder cms = new StringBuilder();
+ for (Map.Entry<String, String> entry : metas.entrySet()) {
+ cms.append(" -m ");
+ cms.append(entry.getKey());
+ cms.append(":");
+ cms.append(entry.getValue());
+ cms.append(" ");
}
- return formatter.toString();
+ return cms.toString();
}
-}
+}
\ No newline at end of file
diff --git a/utils/src/test/java/com/cloud/utils/SwiftUtilTest.java b/utils/src/test/java/com/cloud/utils/SwiftUtilTest.java
index 20c1623d822..0bec6513174 100644
--- a/utils/src/test/java/com/cloud/utils/SwiftUtilTest.java
+++ b/utils/src/test/java/com/cloud/utils/SwiftUtilTest.java
@@ -19,9 +19,15 @@
package com.cloud.utils;
-
-import org.junit.Test;
-import org.mockito.Mockito;
+import static org.mockito.BDDMockito.given;
+import static org.mockito.BDDMockito.mock;
+import static org.junit.Assert.assertThat;
+import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.equalTo;
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.Mockito.when;
import java.io.File;
import java.net.URL;
@@ -29,10 +35,10 @@
import java.security.NoSuchAlgorithmException;
import java.security.SignatureException;
-import static org.junit.Assert.assertArrayEquals;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-import static org.mockito.Mockito.when;
+import org.junit.Test;
+import org.mockito.Mockito;
+
+import com.cloud.utils.SwiftUtil.SwiftClientCfg;
public class SwiftUtilTest {
@@ -90,4 +96,134 @@ public void testGetContainerName(){
assertEquals(expected, output);
}
+
+ @Test
+ public void testGetSwiftCmd(){
+ SwiftClientCfg cfg = mock(SwiftClientCfg.class);
+ given(cfg.getEndPoint()).willReturn("swift.endpoint");
+ given(cfg.getAccount()).willReturn("cs");
+ given(cfg.getUserName()).willReturn("sec-storage");
+ given(cfg.getKey()).willReturn("mypassword");
+ given(cfg.getStoragePolicy()).willReturn(null);
+
+ String cmd = SwiftUtil.getSwiftCmd(cfg, "swift", "stat");
+
+ String expected ="/usr/bin/python swift -A swift.endpoint -U cs:sec-storage -K mypassword stat";
+ assertThat(cmd, is(equalTo(expected)));
+ }
+
+ @Test
+ public void testGetSwiftObjectCmd(){
+ SwiftClientCfg cfg = mock(SwiftClientCfg.class);
+ given(cfg.getEndPoint()).willReturn("swift.endpoint");
+ given(cfg.getAccount()).willReturn("cs");
+ given(cfg.getUserName()).willReturn("sec-storage");
+ given(cfg.getKey()).willReturn("mypassword");
+ given(cfg.getStoragePolicy()).willReturn(null);
+
+ String objectCmd = SwiftUtil.getSwiftObjectCmd(cfg, "swift", "delete", "T-123", "template.vhd");
+
+ String expected ="/usr/bin/python swift -A swift.endpoint -U cs:sec-storage -K mypassword delete T-123 template.vhd";
+ assertThat(objectCmd, is(equalTo(expected)));
+ }
+
+ @Test
+ public void testGetSwiftContainerCmd(){
+ SwiftClientCfg cfg = mock(SwiftClientCfg.class);
+ given(cfg.getEndPoint()).willReturn("swift.endpoint");
+ given(cfg.getAccount()).willReturn("cs");
+ given(cfg.getUserName()).willReturn("sec-storage");
+ given(cfg.getKey()).willReturn("mypassword");
+ given(cfg.getStoragePolicy()).willReturn(null);
+
+ String containerCmd = SwiftUtil.getSwiftContainerCmd(cfg, "swift", "list", "T-123");
+
+ String expected ="/usr/bin/python swift -A swift.endpoint -U cs:sec-storage -K mypassword list T-123";
+ assertThat(containerCmd, is(equalTo(expected)));
+ }
+
+ @Test
+ public void testGetUploadCmd(){
+ SwiftClientCfg cfg = mock(SwiftClientCfg.class);
+ given(cfg.getEndPoint()).willReturn("swift.endpoint");
+ given(cfg.getAccount()).willReturn("cs");
+ given(cfg.getUserName()).willReturn("sec-storage");
+ given(cfg.getKey()).willReturn("mypassword");
+ given(cfg.getStoragePolicy()).willReturn(null);
+
+ String uploadCmd = SwiftUtil.getUploadObjectCommand(cfg,"swift","T-1","template.vhd", 1024);
+
+ String expected ="/usr/bin/python swift -A swift.endpoint -U cs:sec-storage -K mypassword upload T-1 template.vhd";
+ assertThat(uploadCmd, is(equalTo(expected)));
+ }
+
+ @Test
+ public void testGetUploadCmdWithSegmentsBecauseOfSize(){
+ SwiftClientCfg cfg = mock(SwiftClientCfg.class);
+ given(cfg.getEndPoint()).willReturn("swift.endpoint");
+ given(cfg.getAccount()).willReturn("cs");
+ given(cfg.getUserName()).willReturn("sec-storage");
+ given(cfg.getKey()).willReturn("mypassword");
+ given(cfg.getStoragePolicy()).willReturn(null);
+
+ String uploadCmd = SwiftUtil.getUploadObjectCommand(cfg,"swift","T-1","template.vhd", 5368709121L);
+
+ String expected ="/usr/bin/python swift -A swift.endpoint -U cs:sec-storage -K mypassword upload T-1 template.vhd -S 5368709120";
+ assertThat(uploadCmd, is(equalTo(expected)));
+ }
+
+ @Test
+ public void testGetUploadCmdWithStoragePolicy(){
+ SwiftClientCfg cfg = mock(SwiftClientCfg.class);
+ given(cfg.getEndPoint()).willReturn("swift.endpoint");
+ given(cfg.getAccount()).willReturn("cs");
+ given(cfg.getUserName()).willReturn("sec-storage");
+ given(cfg.getKey()).willReturn("mypassword");
+ given(cfg.getStoragePolicy()).willReturn("policy1");
+
+ String uploadCmd = SwiftUtil.getUploadObjectCommand(cfg,"swift","T-1","template.vhd", 1024L);
+ String expected ="/usr/bin/python swift -A swift.endpoint -U cs:sec-storage -K mypassword upload T-1 template.vhd --storage-policy \"policy1\"";
+ assertThat(uploadCmd, is(equalTo(expected)));
+ }
+
+ @Test
+ public void testGetUploadCmdWithSegmentsAndStoragePolicy(){
+ SwiftClientCfg cfg = mock(SwiftClientCfg.class);
+ given(cfg.getEndPoint()).willReturn("swift.endpoint");
+ given(cfg.getAccount()).willReturn("cs");
+ given(cfg.getUserName()).willReturn("sec-storage");
+ given(cfg.getKey()).willReturn("mypassword");
+ given(cfg.getStoragePolicy()).willReturn("policy1");
+ String uploadCmd = SwiftUtil.getUploadObjectCommand(cfg,"swift","T-1","template.vhd", 5368709121L);
+ String expected ="/usr/bin/python swift -A swift.endpoint -U cs:sec-storage -K mypassword upload T-1 template.vhd --storage-policy \"policy1\" -S 5368709120";
+ assertThat(uploadCmd, is(equalTo(expected)));
+ }
+
+ @Test
+ public void testListContainerCmdWithStoragePolicyButNotSupportedByOperation(){
+ SwiftClientCfg cfg = mock(SwiftClientCfg.class);
+ given(cfg.getEndPoint()).willReturn("swift.endpoint");
+ given(cfg.getAccount()).willReturn("cs");
+ given(cfg.getUserName()).willReturn("sec-storage");
+ given(cfg.getKey()).willReturn("mypassword");
+ given(cfg.getStoragePolicy()).willReturn("policy1");
+
+ String uploadCmd = SwiftUtil.getSwiftContainerCmd(cfg, "swift", "list", "T-1");
+ String expected ="/usr/bin/python swift -A swift.endpoint -U cs:sec-storage -K mypassword list T-1";
+ assertThat(uploadCmd, is(equalTo(expected)));
+ }
+
+ @Test
+ public void testListContainerCmdWithoutStoragePolicy(){
+ SwiftClientCfg cfg = mock(SwiftClientCfg.class);
+ given(cfg.getEndPoint()).willReturn("swift.endpoint");
+ given(cfg.getAccount()).willReturn("cs");
+ given(cfg.getUserName()).willReturn("sec-storage");
+ given(cfg.getKey()).willReturn("mypassword");
+ given(cfg.getStoragePolicy()).willReturn(null);
+
+ String uploadCmd = SwiftUtil.getSwiftContainerCmd(cfg, "swift", "list", "T-1");
+ String expected ="/usr/bin/python swift -A swift.endpoint -U cs:sec-storage -K mypassword list T-1";
+ assertThat(uploadCmd, is(equalTo(expected)));
+ }
}
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services