You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cloudstack.apache.org by GitBox <gi...@apache.org> on 2018/04/25 22:42:17 UTC
[GitHub] rhtyd closed pull request #2412: CLOUDSTACK-9677: Adding storage policy support for swift as secondary…
rhtyd closed pull request #2412: CLOUDSTACK-9677: Adding storage policy support for swift as secondary…
URL: https://github.com/apache/cloudstack/pull/2412
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/api/src/com/cloud/agent/api/to/SwiftTO.java b/api/src/com/cloud/agent/api/to/SwiftTO.java
index c7a98660302..b89dfea40e0 100644
--- a/api/src/com/cloud/agent/api/to/SwiftTO.java
+++ b/api/src/com/cloud/agent/api/to/SwiftTO.java
@@ -26,17 +26,19 @@
String userName;
String key;
+ String storagePolicy;
private static final String pathSeparator = "/";
public SwiftTO() {
}
- public SwiftTO(Long id, String url, String account, String userName, String key) {
+ public SwiftTO(Long id, String url, String account, String userName, String key, String storagePolicy) {
this.id = id;
this.url = url;
this.account = account;
this.userName = userName;
this.key = key;
+ this.storagePolicy = storagePolicy;
}
public Long getId() {
@@ -63,6 +65,11 @@ public String getKey() {
return key;
}
+ @Override
+ public String getStoragePolicy() {
+ return this.storagePolicy;
+ }
+
@Override
public DataStoreRole getRole() {
return DataStoreRole.Image;
diff --git a/api/src/org/apache/cloudstack/api/ApiConstants.java b/api/src/org/apache/cloudstack/api/ApiConstants.java
index d9090b89217..dfe9b30f306 100644
--- a/api/src/org/apache/cloudstack/api/ApiConstants.java
+++ b/api/src/org/apache/cloudstack/api/ApiConstants.java
@@ -287,6 +287,7 @@
public static final String STATE = "state";
public static final String STATUS = "status";
public static final String STORAGE_TYPE = "storagetype";
+ public static final String STORAGE_POLICY = "storagepolicy";
public static final String STORAGE_MOTION_ENABLED = "storagemotionenabled";
public static final String STORAGE_CAPABILITIES = "storagecapabilities";
public static final String SYSTEM_VM_TYPE = "systemvmtype";
diff --git a/plugins/hypervisors/xenserver/src/com/cloud/hypervisor/xenserver/resource/XenServerStorageProcessor.java b/plugins/hypervisors/xenserver/src/com/cloud/hypervisor/xenserver/resource/XenServerStorageProcessor.java
index 257c6a23e41..336c32ce159 100644
--- a/plugins/hypervisors/xenserver/src/com/cloud/hypervisor/xenserver/resource/XenServerStorageProcessor.java
+++ b/plugins/hypervisors/xenserver/src/com/cloud/hypervisor/xenserver/resource/XenServerStorageProcessor.java
@@ -18,23 +18,24 @@
*/
package com.cloud.hypervisor.xenserver.resource;
-import com.cloud.agent.api.Answer;
-import com.cloud.agent.api.to.DataObjectType;
-import com.cloud.agent.api.to.DataStoreTO;
-import com.cloud.agent.api.to.DataTO;
-import com.cloud.agent.api.to.DiskTO;
-import com.cloud.agent.api.to.NfsTO;
-import com.cloud.agent.api.to.S3TO;
-import com.cloud.agent.api.to.SwiftTO;
-import com.cloud.exception.InternalErrorException;
-import com.cloud.hypervisor.Hypervisor.HypervisorType;
-import com.cloud.hypervisor.xenserver.resource.CitrixResourceBase.SRType;
-import com.cloud.storage.DataStoreRole;
-import com.cloud.storage.Storage;
-import com.cloud.storage.Storage.ImageFormat;
-import com.cloud.storage.resource.StorageProcessor;
-import com.cloud.utils.exception.CloudRuntimeException;
-import com.cloud.utils.storage.S3.ClientOptions;
+import static com.cloud.utils.ReflectUtil.flattenProperties;
+import static com.google.common.collect.Lists.newArrayList;
+
+import java.io.File;
+import java.net.URI;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+
+import org.apache.commons.lang3.BooleanUtils;
+import org.apache.log4j.Logger;
+import org.apache.xmlrpc.XmlRpcException;
+
+import com.google.common.annotations.VisibleForTesting;
import com.xensource.xenapi.Connection;
import com.xensource.xenapi.SR;
import com.xensource.xenapi.Types;
@@ -44,6 +45,7 @@
import com.xensource.xenapi.VBD;
import com.xensource.xenapi.VDI;
import com.xensource.xenapi.VM;
+
import org.apache.cloudstack.agent.directdownload.DirectDownloadCommand;
import org.apache.cloudstack.storage.command.AttachAnswer;
import org.apache.cloudstack.storage.command.AttachCommand;
@@ -65,20 +67,24 @@
import org.apache.cloudstack.storage.to.SnapshotObjectTO;
import org.apache.cloudstack.storage.to.TemplateObjectTO;
import org.apache.cloudstack.storage.to.VolumeObjectTO;
-import org.apache.log4j.Logger;
-import org.apache.xmlrpc.XmlRpcException;
-
-import java.io.File;
-import java.net.URI;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.UUID;
-import static com.cloud.utils.ReflectUtil.flattenProperties;
-import static com.google.common.collect.Lists.newArrayList;
+import com.cloud.agent.api.Answer;
+import com.cloud.agent.api.to.DataObjectType;
+import com.cloud.agent.api.to.DataStoreTO;
+import com.cloud.agent.api.to.DataTO;
+import com.cloud.agent.api.to.DiskTO;
+import com.cloud.agent.api.to.NfsTO;
+import com.cloud.agent.api.to.S3TO;
+import com.cloud.agent.api.to.SwiftTO;
+import com.cloud.exception.InternalErrorException;
+import com.cloud.hypervisor.Hypervisor.HypervisorType;
+import com.cloud.hypervisor.xenserver.resource.CitrixResourceBase.SRType;
+import com.cloud.storage.DataStoreRole;
+import com.cloud.storage.Storage;
+import com.cloud.storage.Storage.ImageFormat;
+import com.cloud.storage.resource.StorageProcessor;
+import com.cloud.utils.exception.CloudRuntimeException;
+import com.cloud.utils.storage.S3.ClientOptions;
public class XenServerStorageProcessor implements StorageProcessor {
private static final Logger s_logger = Logger.getLogger(XenServerStorageProcessor.class);
@@ -914,20 +920,55 @@ public Answer copyVolumeFromPrimaryToSecondary(final CopyCommand cmd) {
private boolean swiftUpload(final Connection conn, final SwiftTO swift, final String container, final String ldir, final String lfilename, final Boolean isISCSI,
final int wait) {
- String result = null;
+
+ List<String> params = getSwiftParams(swift, container, ldir, lfilename, isISCSI);
+
try {
- result =
- hypervisorResource.callHostPluginAsync(conn, "swiftxenserver", "swift", wait, "op", "upload", "url", swift.getUrl(), "account", swift.getAccount(), "username",
- swift.getUserName(), "key", swift.getKey(), "container", container, "ldir", ldir, "lfilename", lfilename, "isISCSI", isISCSI.toString());
- if (result != null && result.equals("true")) {
- return true;
- }
+ String result = hypervisorResource.callHostPluginAsync(conn, "swiftxenserver", "swift", wait, params.toArray(new String[params.size()]));
+ return BooleanUtils.toBoolean(result);
} catch (final Exception e) {
s_logger.warn("swift upload failed due to " + e.toString(), e);
}
return false;
}
+ @VisibleForTesting
+ List<String> getSwiftParams(SwiftTO swift, String container, String ldir, String lfilename, Boolean isISCSI) {
+ // ORDER IS IMPORTANT
+ List<String> params = new ArrayList<>();
+
+ //operation
+ params.add("op");
+ params.add("upload");
+
+ //auth
+ params.add("url");
+ params.add(swift.getUrl());
+ params.add("account");
+ params.add(swift.getAccount());
+ params.add("username");
+ params.add(swift.getUserName());
+ params.add("key");
+ params.add(swift.getKey());
+
+ // object info
+ params.add("container");
+ params.add(container);
+ params.add("ldir");
+ params.add(ldir);
+ params.add("lfilename");
+ params.add(lfilename);
+ params.add("isISCSI");
+ params.add(isISCSI.toString());
+
+ if (swift.getStoragePolicy() != null) {
+ params.add("storagepolicy");
+ params.add(swift.getStoragePolicy());
+ }
+
+ return params;
+ }
+
protected String deleteSnapshotBackup(final Connection conn, final String localMountPoint, final String path, final String secondaryStorageMountPath, final String backupUUID) {
// If anybody modifies the formatting below again, I'll skin them
diff --git a/plugins/hypervisors/xenserver/test/com/cloud/hypervisor/xenserver/resource/XenServerStorageProcessorTest.java b/plugins/hypervisors/xenserver/test/com/cloud/hypervisor/xenserver/resource/XenServerStorageProcessorTest.java
new file mode 100644
index 00000000000..75ac8b8ad4d
--- /dev/null
+++ b/plugins/hypervisors/xenserver/test/com/cloud/hypervisor/xenserver/resource/XenServerStorageProcessorTest.java
@@ -0,0 +1,113 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package com.cloud.hypervisor.xenserver.resource;
+
+import static org.mockito.Mockito.when;
+
+import java.util.List;
+
+import org.junit.Assert;
+import org.junit.Test;
+import org.mockito.Mockito;
+
+import com.cloud.agent.api.to.SwiftTO;
+
+public class XenServerStorageProcessorTest {
+ @Test
+ public void testOrderOfSwiftUplodScriptParamsWithoutStoragePolicy() {
+ CitrixResourceBase resource = Mockito.mock(CitrixResourceBase.class);
+ XenServerStorageProcessor mock = new XenServerStorageProcessor(resource);
+
+ SwiftTO swift = Mockito.mock(SwiftTO.class);
+ when(swift.getStoragePolicy()).thenReturn(null);
+
+ String container = "sample-container-name";
+ String ldir = "sample-ldir";
+ String lfilename = "sample-lfilename";
+ Boolean isISCSI = true;
+
+ List<String> params = mock.getSwiftParams(swift, container, ldir, lfilename, isISCSI);
+
+ // make sure the params not null and has correct number of items in it
+ Assert.assertNotNull("params is null", params);
+ Assert.assertTrue("Expected param list size is 18 but it was" + params.size(), params.size() == 18);
+
+ // check the order of params
+ Assert.assertEquals("unexpected param.", "op", params.get(0));
+ Assert.assertEquals("unexpected param.", "upload", params.get(1));
+ Assert.assertEquals("unexpected param.", "url", params.get(2));
+ Assert.assertEquals("unexpected param.", swift.getUrl(), params.get(3));
+ Assert.assertEquals("unexpected param.", "account", params.get(4));
+ Assert.assertEquals("unexpected param.", swift.getAccount(), params.get(5));
+ Assert.assertEquals("unexpected param.", "username", params.get(6));
+ Assert.assertEquals("unexpected param.", swift.getUserName(), params.get(7));
+ Assert.assertEquals("unexpected param.", "key", params.get(8));
+ Assert.assertEquals("unexpected param.", swift.getKey(), params.get(9));
+ Assert.assertEquals("unexpected param.", "container", params.get(10));
+ Assert.assertEquals("unexpected param.", container, params.get(11));
+ Assert.assertEquals("unexpected param.", "ldir", params.get(12));
+ Assert.assertEquals("unexpected param.", ldir, params.get(13));
+ Assert.assertEquals("unexpected param.", "lfilename", params.get(14));
+ Assert.assertEquals("unexpected param.", lfilename, params.get(15));
+ Assert.assertEquals("unexpected param.", "isISCSI", params.get(16));
+ Assert.assertEquals("unexpected param.", isISCSI.toString(), params.get(17));
+ }
+
+ @Test
+ public void testOrderOfSwiftUplodScriptParamsWithStoragePolicy() {
+ CitrixResourceBase resource = Mockito.mock(CitrixResourceBase.class);
+ XenServerStorageProcessor mock = new XenServerStorageProcessor(resource);
+
+ SwiftTO swift = Mockito.mock(SwiftTO.class);
+ when(swift.getStoragePolicy()).thenReturn("sample-storagepolicy");
+
+ String container = "sample-container-name";
+ String ldir = "sample-ldir";
+ String lfilename = "sample-lfilename";
+ Boolean isISCSI = true;
+
+ List<String> params = mock.getSwiftParams(swift, container, ldir, lfilename, isISCSI);
+
+ // make sure the params not null and has correct number of items in it
+ Assert.assertNotNull("params is null", params);
+ Assert.assertTrue("Expected param list size is 20 but it was" + params.size(), params.size() == 20);
+
+ // check the order of params
+ Assert.assertEquals("unexpected param.", "op", params.get(0));
+ Assert.assertEquals("unexpected param.", "upload", params.get(1));
+ Assert.assertEquals("unexpected param.", "url", params.get(2));
+ Assert.assertEquals("unexpected param.", swift.getUrl(), params.get(3));
+ Assert.assertEquals("unexpected param.", "account", params.get(4));
+ Assert.assertEquals("unexpected param.", swift.getAccount(), params.get(5));
+ Assert.assertEquals("unexpected param.", "username", params.get(6));
+ Assert.assertEquals("unexpected param.", swift.getUserName(), params.get(7));
+ Assert.assertEquals("unexpected param.", "key", params.get(8));
+ Assert.assertEquals("unexpected param.", swift.getKey(), params.get(9));
+ Assert.assertEquals("unexpected param.", "container", params.get(10));
+ Assert.assertEquals("unexpected param.", container, params.get(11));
+ Assert.assertEquals("unexpected param.", "ldir", params.get(12));
+ Assert.assertEquals("unexpected param.", ldir, params.get(13));
+ Assert.assertEquals("unexpected param.", "lfilename", params.get(14));
+ Assert.assertEquals("unexpected param.", lfilename, params.get(15));
+ Assert.assertEquals("unexpected param.", "isISCSI", params.get(16));
+ Assert.assertEquals("unexpected param.", isISCSI.toString(), params.get(17));
+ Assert.assertEquals("unexpected param.", "storagepolicy", params.get(18));
+ Assert.assertEquals("unexpected param.", "sample-storagepolicy", params.get(19));
+ }
+}
diff --git a/plugins/storage/image/swift/src/org/apache/cloudstack/storage/datastore/driver/SwiftImageStoreDriverImpl.java b/plugins/storage/image/swift/src/org/apache/cloudstack/storage/datastore/driver/SwiftImageStoreDriverImpl.java
index 2816c6063f6..7e1486214bc 100644
--- a/plugins/storage/image/swift/src/org/apache/cloudstack/storage/datastore/driver/SwiftImageStoreDriverImpl.java
+++ b/plugins/storage/image/swift/src/org/apache/cloudstack/storage/datastore/driver/SwiftImageStoreDriverImpl.java
@@ -68,7 +68,7 @@
public DataStoreTO getStoreTO(DataStore store) {
ImageStoreImpl imgStore = (ImageStoreImpl)store;
Map<String, String> details = _imageStoreDetailsDao.getDetails(imgStore.getId());
- return new SwiftTO(imgStore.getId(), imgStore.getUri(), details.get(ApiConstants.ACCOUNT), details.get(ApiConstants.USERNAME), details.get(ApiConstants.KEY));
+ return new SwiftTO(imgStore.getId(), imgStore.getUri(), details.get(ApiConstants.ACCOUNT), details.get(ApiConstants.USERNAME), details.get(ApiConstants.KEY), details.get(ApiConstants.STORAGE_POLICY));
}
@Override
diff --git a/scripts/storage/secondary/swift b/scripts/storage/secondary/swift
index 4138db8b17e..c09c53037f8 100755
--- a/scripts/storage/secondary/swift
+++ b/scripts/storage/secondary/swift
@@ -1473,8 +1473,9 @@ post [options] [container] [object]
Updates meta information for the account, container, or object depending on
the args given. If the container is not found, it will be created
automatically; but this is not true for accounts and objects. Containers
- also allow the -r (or --read-acl) and -w (or --write-acl) options. The -m
- or --meta option is allowed on all and used to define the user meta data
+ also allow the -r (or --read-acl) and -w (or --write-acl) options.
+ The --storage-policy will set a storage policy to the container if the container does not exist.
+ The -m or --meta option is allowed on all and used to define the user meta data
items to set in the form Name:Value. This option can be repeated. Example:
post -m Color:Blue -m Size:Large'''.strip('\n')
@@ -1493,6 +1494,8 @@ def st_post(options, args, print_queue, error_queue):
parser.add_option('-m', '--meta', action='append', dest='meta', default=[],
help='Sets a meta data item with the syntax name:value. This option '
'may be repeated. Example: -m Color:Blue -m Size:Large')
+ parser.add_option('', '--storage-policy', action='store', dest='storage_policy',
+ help='Sets a storage policy to the container if the container does not exist')
(options, args) = parse_args(parser, args)
args = args[1:]
if (options.read_acl or options.write_acl or options.sync_to or
@@ -1529,6 +1532,8 @@ def st_post(options, args, print_queue, error_queue):
headers['X-Container-Sync-To'] = options.sync_to
if options.sync_key is not None:
headers['X-Container-Sync-Key'] = options.sync_key
+ if options.storage_policy is not None:
+ headers['X-Storage-Policy'] = options.storage_policy
try:
conn.post_container(args[0], headers=headers)
except ClientException, err:
@@ -1558,7 +1563,8 @@ upload [options] container file_or_directory [file_or_directory] [...]
Uploads to the given container the files and directories specified by the
remaining args. -c or --changed is an option that will only upload files
that have changed since the last upload. -S <size> or --segment-size <size>
- and --leave-segments are options as well (see --help for more).
+ and --leave-segments are options as well (see --help for more). --storage-policy
+ Sets a storage policy to the container if the container does not exist.
'''.strip('\n')
@@ -1576,6 +1582,8 @@ def st_upload(options, args, print_queue, error_queue):
dest='leave_segments', default=False, help='Indicates that you want '
'the older segments of manifest objects left alone (in the case of '
'overwrites)')
+ parser.add_option('', '--storage-policy', action='store', dest='storage_policy',
+ help='Sets a storage policy to the container if the container does not exist')
(options, args) = parse_args(parser, args)
args = args[1:]
if len(args) < 2:
@@ -1749,9 +1757,12 @@ def st_upload(options, args, print_queue, error_queue):
# permissions, so we'll ignore any error. If there's really a problem,
# it'll surface on the first object PUT.
try:
- conn.put_container(args[0])
+ container_headers = {}
+ if options.storage_policy is not None:
+ container_headers['X-Storage-Policy'] = options.storage_policy
+ conn.put_container(args[0],headers=container_headers)
if options.segment_size is not None:
- conn.put_container(args[0] + '_segments')
+ conn.put_container(args[0] + '_segments',headers=container_headers)
except Exception:
pass
try:
diff --git a/scripts/vm/hypervisor/xenserver/swift b/scripts/vm/hypervisor/xenserver/swift
index c9d2cebb04f..603bfdc6bce 100755
--- a/scripts/vm/hypervisor/xenserver/swift
+++ b/scripts/vm/hypervisor/xenserver/swift
@@ -1475,8 +1475,9 @@ post [options] [container] [object]
Updates meta information for the account, container, or object depending on
the args given. If the container is not found, it will be created
automatically; but this is not true for accounts and objects. Containers
- also allow the -r (or --read-acl) and -w (or --write-acl) options. The -m
- or --meta option is allowed on all and used to define the user meta data
+ also allow the -r (or --read-acl) and -w (or --write-acl) options.
+ The --storage-policy will set a storage policy to the container if the container does not exist.
+ The -m or --meta option is allowed on all and used to define the user meta data
items to set in the form Name:Value. This option can be repeated. Example:
post -m Color:Blue -m Size:Large'''.strip('\n')
@@ -1495,6 +1496,8 @@ def st_post(options, args, print_queue, error_queue):
parser.add_option('-m', '--meta', action='append', dest='meta', default=[],
help='Sets a meta data item with the syntax name:value. This option '
'may be repeated. Example: -m Color:Blue -m Size:Large')
+ parser.add_option('', '--storage-policy', action='store', dest='storage_policy',
+ help='Sets a storage policy to the container if the container does not exist')
(options, args) = parse_args(parser, args)
args = args[1:]
if (options.read_acl or options.write_acl or options.sync_to or
@@ -1531,6 +1534,8 @@ def st_post(options, args, print_queue, error_queue):
headers['X-Container-Sync-To'] = options.sync_to
if options.sync_key is not None:
headers['X-Container-Sync-Key'] = options.sync_key
+ if options.storage_policy is not None:
+ headers['X-Storage-Policy'] = options.storage_policy
try:
conn.post_container(args[0], headers=headers)
except ClientException, err:
@@ -1560,7 +1565,8 @@ upload [options] container file_or_directory [file_or_directory] [...]
Uploads to the given container the files and directories specified by the
remaining args. -c or --changed is an option that will only upload files
that have changed since the last upload. -S <size> or --segment-size <size>
- and --leave-segments are options as well (see --help for more).
+ and --leave-segments are options as well (see --help for more). --storage-policy
+ Sets a storage policy to the container if the container does not exist.
'''.strip('\n')
@@ -1578,6 +1584,8 @@ def st_upload(options, args, print_queue, error_queue):
dest='leave_segments', default=False, help='Indicates that you want '
'the older segments of manifest objects left alone (in the case of '
'overwrites)')
+ parser.add_option('', '--storage-policy', action='store', dest='storage_policy',
+ help='Sets a storage policy to the container if the container does not exist')
(options, args) = parse_args(parser, args)
args = args[1:]
if len(args) < 2:
@@ -1751,9 +1759,12 @@ def st_upload(options, args, print_queue, error_queue):
# permissions, so we'll ignore any error. If there's really a problem,
# it'll surface on the first object PUT.
try:
- conn.put_container(args[0])
+ container_headers = {}
+ if options.storage_policy is not None:
+ container_headers['X-Storage-Policy'] = options.storage_policy
+ conn.put_container(args[0],headers=container_headers)
if options.segment_size is not None:
- conn.put_container(args[0] + '_segments')
+ conn.put_container(args[0] + '_segments',headers=container_headers)
except Exception:
pass
try:
diff --git a/ui/l10n/en.js b/ui/l10n/en.js
index 7b19946b1ed..26745503370 100644
--- a/ui/l10n/en.js
+++ b/ui/l10n/en.js
@@ -1616,6 +1616,7 @@ var dictionary = {"ICMP.code":"ICMP Code",
"label.storage.tags":"Storage Tags",
"label.storage.traffic":"Storage Traffic",
"label.storage.type":"Storage Type",
+"label.storagepolicy":"Storage policy",
"label.subdomain.access":"Subdomain Access",
"label.submit":"Submit",
"label.submitted.by":"[Submitted by: <span id=\"submitted_by\"></span>]",
diff --git a/ui/scripts/system.js b/ui/scripts/system.js
index 07520df1d46..17c78d38760 100755
--- a/ui/scripts/system.js
+++ b/ui/scripts/system.js
@@ -19845,6 +19845,7 @@
$form.find('.form-item[rel=account]').hide();
$form.find('.form-item[rel=username]').hide();
$form.find('.form-item[rel=key]').hide();
+ $form.find('.form-item[rel=storagepolicy]').hide();
} else if ($(this).val() == "SMB") {
//NFS, SMB
$form.find('.form-item[rel=zoneid]').css('display', 'inline-block');
@@ -19877,6 +19878,7 @@
$form.find('.form-item[rel=account]').hide();
$form.find('.form-item[rel=username]').hide();
$form.find('.form-item[rel=key]').hide();
+ $form.find('.form-item[rel=storagepolicy]').hide();
} else if ($(this).val() == "S3") {
//NFS, SMB
$form.find('.form-item[rel=zoneid]').hide();
@@ -19911,6 +19913,7 @@
$form.find('.form-item[rel=account]').hide();
$form.find('.form-item[rel=username]').hide();
$form.find('.form-item[rel=key]').hide();
+ $form.find('.form-item[rel=storagepolicy]').hide();
} else if ($(this).val() == "Swift") {
//NFS, SMB
$form.find('.form-item[rel=zoneid]').hide();
@@ -19943,6 +19946,7 @@
$form.find('.form-item[rel=account]').css('display', 'inline-block');
$form.find('.form-item[rel=username]').css('display', 'inline-block');
$form.find('.form-item[rel=key]').css('display', 'inline-block');
+ $form.find('.form-item[rel=storagepolicy]').css('display', 'inline-block');
}
});
@@ -20133,14 +20137,26 @@
}
},
account: {
- label: 'label.account'
+ label: 'label.account',
+ validation: {
+ required: true
+ }
},
username: {
- label: 'label.username'
+ label: 'label.username',
+ validation: {
+ required: true
+ }
},
key: {
- label: 'label.key'
- }
+ label: 'label.key',
+ validation: {
+ required: true
+ }
+ },
+ storagepolicy: {
+ label: 'label.storagepolicy'
+ }
//Swift (end)
}
},
@@ -20307,6 +20323,11 @@
data[ 'details[' + index.toString() + '].value'] = args.data.key;
index++;
}
+ if (args.data.storagepolicy != null && args.data.storagepolicy.length > 0) {
+ data[ 'details[' + index.toString() + '].key'] = 'storagepolicy';
+ data[ 'details[' + index.toString() + '].value'] = args.data.storagepolicy;
+ index++;
+ }
$.ajax({
url: createURL('addImageStore'),
data: data,
diff --git a/utils/src/main/java/com/cloud/utils/SwiftUtil.java b/utils/src/main/java/com/cloud/utils/SwiftUtil.java
index ce1bee36b62..34aceb51553 100644
--- a/utils/src/main/java/com/cloud/utils/SwiftUtil.java
+++ b/utils/src/main/java/com/cloud/utils/SwiftUtil.java
@@ -24,26 +24,29 @@
import java.security.InvalidKeyException;
import java.security.NoSuchAlgorithmException;
import java.security.SignatureException;
-import java.util.Map;
import java.util.Arrays;
import java.util.HashMap;
-import java.util.Formatter;
+import java.util.Map;
+import javax.crypto.Mac;
+import javax.crypto.spec.SecretKeySpec;
+
+import org.apache.commons.codec.binary.Hex;
import org.apache.log4j.Logger;
import com.cloud.utils.exception.CloudRuntimeException;
import com.cloud.utils.script.OutputInterpreter;
import com.cloud.utils.script.Script;
-import javax.crypto.Mac;
-import javax.crypto.spec.SecretKeySpec;
-
public class SwiftUtil {
private static Logger logger = Logger.getLogger(SwiftUtil.class);
- private static final long SWIFT_MAX_SIZE = 5L * 1024L * 1024L * 1024L;
+ protected static final long SWIFT_MAX_SIZE = 5L * 1024L * 1024L * 1024L;
private static final String HMAC_SHA1_ALGORITHM = "HmacSHA1";
-
-
+ private static final String CD_SRC = "cd %s;";
+ private static final String SWIFT_CMD= "/usr/bin/python %s -A %s -U %s:%s -K %s %s";
+ private static final String WITH_STORAGE_POLICY = " --storage-policy \"%s\"";
+ private static final String WITH_SEGMENTS = " -S "+SWIFT_MAX_SIZE;
+ private static final String[] OPERATIONS_WITH_STORAGE_POLICIES = {"post","upload"};
public interface SwiftClientCfg {
String getAccount();
@@ -53,6 +56,8 @@
String getKey();
String getEndPoint();
+
+ String getStoragePolicy();
}
private static String getSwiftCLIPath() {
@@ -65,19 +70,10 @@ private static String getSwiftCLIPath() {
}
public static boolean postMeta(SwiftClientCfg cfg, String container, String object, Map<String, String> metas) {
- String swiftCli = getSwiftCLIPath();
- StringBuilder cms = new StringBuilder();
- for (Map.Entry<String, String> entry : metas.entrySet()) {
- cms.append(" -m ");
- cms.append(entry.getKey());
- cms.append(":");
- cms.append(entry.getValue());
- cms.append(" ");
- }
Script command = new Script("/bin/bash", logger);
command.add("-c");
- command.add("/usr/bin/python " + swiftCli + " -A " + cfg.getEndPoint() + " -U " + cfg.getAccount() + ":" + cfg.getUserName() + " -K " + cfg.getKey() + " post " +
- container + " " + object + " " + cms.toString());
+ command.add(getSwiftObjectCmd(cfg, getSwiftCLIPath(),"post", container, object) + getMeta(metas));
+
OutputInterpreter.OneLineParser parser = new OutputInterpreter.OneLineParser();
String result = command.execute(parser);
if (result != null) {
@@ -87,21 +83,14 @@ public static boolean postMeta(SwiftClientCfg cfg, String container, String obje
}
public static String putObject(SwiftClientCfg cfg, File srcFile, String container, String fileName) {
- String swiftCli = getSwiftCLIPath();
if (fileName == null) {
fileName = srcFile.getName();
}
- String srcDirectory = srcFile.getParent();
+
Script command = new Script("/bin/bash", logger);
- long size = srcFile.length();
command.add("-c");
- if (size <= SWIFT_MAX_SIZE) {
- command.add("cd " + srcDirectory + ";/usr/bin/python " + swiftCli + " -A " + cfg.getEndPoint() + " -U " + cfg.getAccount() + ":" + cfg.getUserName() +
- " -K " + cfg.getKey() + " upload " + container + " " + fileName);
- } else {
- command.add("cd " + srcDirectory + ";/usr/bin/python " + swiftCli + " -A " + cfg.getEndPoint() + " -U " + cfg.getAccount() + ":" + cfg.getUserName() +
- " -K " + cfg.getKey() + " upload -S " + SWIFT_MAX_SIZE + " " + container + " " + fileName);
- }
+ command.add(String.format(CD_SRC, srcFile.getParent())+getUploadObjectCommand(cfg, getSwiftCLIPath(), container,fileName, srcFile.length()));
+
OutputInterpreter.AllLinesParser parser = new OutputInterpreter.AllLinesParser();
String result = command.execute(parser);
if (result != null) {
@@ -120,38 +109,19 @@ public static String putObject(SwiftClientCfg cfg, File srcFile, String containe
return container + File.separator + srcFile.getName();
}
- private static StringBuilder buildSwiftCmd(SwiftClientCfg swift) {
- String swiftCli = getSwiftCLIPath();
- StringBuilder sb = new StringBuilder();
- sb.append(" /usr/bin/python ");
- sb.append(swiftCli);
- sb.append(" -A ");
- sb.append(swift.getEndPoint());
- sb.append(" -U ");
- sb.append(swift.getAccount());
- sb.append(":");
- sb.append(swift.getUserName());
- sb.append(" -K ");
- sb.append(swift.getKey());
- sb.append(" ");
- return sb;
- }
-
public static String[] list(SwiftClientCfg swift, String container, String rFilename) {
- getSwiftCLIPath();
- Script command = new Script("/bin/bash", logger);
- command.add("-c");
-
- StringBuilder swiftCmdBuilder = buildSwiftCmd(swift);
- swiftCmdBuilder.append(" list ");
- swiftCmdBuilder.append(container);
+ StringBuilder swiftCmdBuilder = new StringBuilder();
+ swiftCmdBuilder.append(getSwiftContainerCmd(swift, getSwiftCLIPath(), "list", container));
if (rFilename != null) {
swiftCmdBuilder.append(" -p ");
swiftCmdBuilder.append(rFilename);
}
+ Script command = new Script("/bin/bash", logger);
+ command.add("-c");
command.add(swiftCmdBuilder.toString());
+
OutputInterpreter.AllLinesParser parser = new OutputInterpreter.AllLinesParser();
String result = command.execute(parser);
if (result == null && parser.getLines() != null && !parser.getLines().equalsIgnoreCase("")) {
@@ -178,11 +148,11 @@ public static File getObject(SwiftClientCfg cfg, File destDirectory, String swif
} else {
destFilePath = destDirectory.getAbsolutePath();
}
- String swiftCli = getSwiftCLIPath();
+
Script command = new Script("/bin/bash", logger);
command.add("-c");
- command.add("/usr/bin/python " + swiftCli + " -A " + cfg.getEndPoint() + " -U " + cfg.getAccount() + ":" + cfg.getUserName() + " -K " + cfg.getKey() +
- " download " + container + " " + srcPath + " -o " + destFilePath);
+ command.add(getSwiftObjectCmd(cfg, getSwiftCLIPath(), "download", container, srcPath)+" -o " + destFilePath);
+
OutputInterpreter.AllLinesParser parser = new OutputInterpreter.AllLinesParser();
String result = command.execute(parser);
if (result != null) {
@@ -203,27 +173,6 @@ public static File getObject(SwiftClientCfg cfg, File destDirectory, String swif
return new File(destFilePath);
}
- public static String getContainerName(String type, Long id) {
- if (type.startsWith("T")) {
- return "T-" + id;
- } else if (type.startsWith("S")) {
- return "S-" + id;
- } else if (type.startsWith("V")) {
- return "V-" + id;
- }
- return null;
- }
-
- public static String[] splitSwiftPath(String path) {
- int index = path.indexOf(File.separator);
- if (index == -1) {
- return null;
- }
- String[] paths = new String[2];
- paths[0] = path.substring(0, index);
- paths[1] = path.substring(index + 1);
- return paths;
- }
public static boolean deleteObject(SwiftClientCfg cfg, String path) {
Script command = new Script("/bin/bash", logger);
@@ -236,13 +185,8 @@ public static boolean deleteObject(SwiftClientCfg cfg, String path) {
String container = paths[0];
String objectName = paths[1];
- StringBuilder swiftCmdBuilder = buildSwiftCmd(cfg);
- swiftCmdBuilder.append(" delete ");
- swiftCmdBuilder.append(container);
- swiftCmdBuilder.append(" ");
- swiftCmdBuilder.append(objectName);
+ command.add(getSwiftObjectCmd(cfg, getSwiftCLIPath(), "delete", container, objectName));
- command.add(swiftCmdBuilder.toString());
OutputInterpreter.AllLinesParser parser = new OutputInterpreter.AllLinesParser();
command.execute(parser);
return true;
@@ -284,7 +228,7 @@ public static URL generateTempUrl(SwiftClientCfg cfg, String container, String o
}
- public static String calculateRFC2104HMAC(String data, String key)
+ static String calculateRFC2104HMAC(String data, String key)
throws SignatureException, NoSuchAlgorithmException, InvalidKeyException {
SecretKeySpec signingKey = new SecretKeySpec(key.getBytes(), HMAC_SHA1_ALGORITHM);
@@ -294,12 +238,75 @@ public static String calculateRFC2104HMAC(String data, String key)
}
- public static String toHexString(byte[] bytes) {
+ static String toHexString(byte[] bytes) {
+ return Hex.encodeHexString(bytes);
+ }
+
+ /////////////// SWIFT CMD STRING HELPERS ///////////////
+ protected static String getSwiftCmd(SwiftClientCfg cfg, String swiftCli, String operation){
+ return String.format(SWIFT_CMD, swiftCli,cfg.getEndPoint(),cfg.getAccount(),cfg.getUserName(),cfg.getKey(),operation);
+ }
+
+ protected static String getSwiftObjectCmd(SwiftClientCfg cfg, String swiftCliPath, String operation,String container, String objectName) {
+ String cmd = getSwiftCmd(cfg,swiftCliPath, operation) +" "+ container+" "+objectName;
+ if(StringUtils.isNotBlank(cfg.getStoragePolicy()) && supportsStoragePolicies(operation)){
+ return cmd + String.format(WITH_STORAGE_POLICY, cfg.getStoragePolicy());
+ }
+ return cmd;
+ }
+
+ private static boolean supportsStoragePolicies(String operation) {
+ for(String supportedOp: OPERATIONS_WITH_STORAGE_POLICIES){
+ if(supportedOp.equals(operation)){
+ return true;
+ }
+ }
+ return false;
+ }
+
+ protected static String getSwiftContainerCmd(SwiftClientCfg cfg, String swiftCliPath, String operation, String container) {
+ return getSwiftCmd(cfg,swiftCliPath, operation) +" "+ container;
+ }
+
+ protected static String getUploadObjectCommand(SwiftClientCfg cfg, String swiftCliPath, String container, String objectName, long size) {
+ String cmd = getSwiftObjectCmd(cfg, swiftCliPath, "upload", container, objectName);
+ if(size > SWIFT_MAX_SIZE){
+ return cmd + WITH_SEGMENTS;
+ }
+ return cmd;
+ }
+
+ public static String getContainerName(String type, Long id) {
+ if (type.startsWith("T")) {
+ return "T-" + id;
+ } else if (type.startsWith("S")) {
+ return "S-" + id;
+ } else if (type.startsWith("V")) {
+ return "V-" + id;
+ }
+ return null;
+ }
+
+ public static String[] splitSwiftPath(String path) {
+ int index = path.indexOf(File.separator);
+ if (index == -1) {
+ return null;
+ }
+ String[] paths = new String[2];
+ paths[0] = path.substring(0, index);
+ paths[1] = path.substring(index + 1);
+ return paths;
+ }
- Formatter formatter = new Formatter();
- for (byte b : bytes) {
- formatter.format("%02x", b);
+ private static String getMeta(Map<String, String> metas) {
+ StringBuilder cms = new StringBuilder();
+ for (Map.Entry<String, String> entry : metas.entrySet()) {
+ cms.append(" -m ");
+ cms.append(entry.getKey());
+ cms.append(":");
+ cms.append(entry.getValue());
+ cms.append(" ");
}
- return formatter.toString();
+ return cms.toString();
}
-}
+}
\ No newline at end of file
diff --git a/utils/src/test/java/com/cloud/utils/SwiftUtilTest.java b/utils/src/test/java/com/cloud/utils/SwiftUtilTest.java
index 20c1623d822..6dc2cc78cac 100644
--- a/utils/src/test/java/com/cloud/utils/SwiftUtilTest.java
+++ b/utils/src/test/java/com/cloud/utils/SwiftUtilTest.java
@@ -19,9 +19,15 @@
package com.cloud.utils;
-
-import org.junit.Test;
-import org.mockito.Mockito;
+import static org.mockito.BDDMockito.given;
+import static org.mockito.BDDMockito.mock;
+import static org.junit.Assert.assertThat;
+import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.equalTo;
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.Mockito.when;
import java.io.File;
import java.net.URL;
@@ -29,10 +35,10 @@
import java.security.NoSuchAlgorithmException;
import java.security.SignatureException;
-import static org.junit.Assert.assertArrayEquals;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-import static org.mockito.Mockito.when;
+import org.junit.Test;
+import org.mockito.Mockito;
+
+import com.cloud.utils.SwiftUtil.SwiftClientCfg;
public class SwiftUtilTest {
@@ -90,4 +96,134 @@ public void testGetContainerName(){
assertEquals(expected, output);
}
+
+ @Test
+ public void testGetSwiftCmd() {
+ SwiftClientCfg cfg = mock(SwiftClientCfg.class);
+ given(cfg.getEndPoint()).willReturn("swift.endpoint");
+ given(cfg.getAccount()).willReturn("cs");
+ given(cfg.getUserName()).willReturn("sec-storage");
+ given(cfg.getKey()).willReturn("mypassword");
+ given(cfg.getStoragePolicy()).willReturn(null);
+
+ String cmd = SwiftUtil.getSwiftCmd(cfg, "swift", "stat");
+
+ String expected = "/usr/bin/python swift -A swift.endpoint -U cs:sec-storage -K mypassword stat";
+ assertThat(cmd, is(equalTo(expected)));
+ }
+
+ @Test
+ public void testGetSwiftObjectCmd() {
+ SwiftClientCfg cfg = mock(SwiftClientCfg.class);
+ given(cfg.getEndPoint()).willReturn("swift.endpoint");
+ given(cfg.getAccount()).willReturn("cs");
+ given(cfg.getUserName()).willReturn("sec-storage");
+ given(cfg.getKey()).willReturn("mypassword");
+ given(cfg.getStoragePolicy()).willReturn(null);
+
+ String objectCmd = SwiftUtil.getSwiftObjectCmd(cfg, "swift", "delete", "T-123", "template.vhd");
+
+ String expected = "/usr/bin/python swift -A swift.endpoint -U cs:sec-storage -K mypassword delete T-123 template.vhd";
+ assertThat(objectCmd, is(equalTo(expected)));
+ }
+
+ @Test
+ public void testGetSwiftContainerCmd() {
+ SwiftClientCfg cfg = mock(SwiftClientCfg.class);
+ given(cfg.getEndPoint()).willReturn("swift.endpoint");
+ given(cfg.getAccount()).willReturn("cs");
+ given(cfg.getUserName()).willReturn("sec-storage");
+ given(cfg.getKey()).willReturn("mypassword");
+ given(cfg.getStoragePolicy()).willReturn(null);
+
+ String containerCmd = SwiftUtil.getSwiftContainerCmd(cfg, "swift", "list", "T-123");
+
+ String expected = "/usr/bin/python swift -A swift.endpoint -U cs:sec-storage -K mypassword list T-123";
+ assertThat(containerCmd, is(equalTo(expected)));
+ }
+
+ @Test
+ public void testGetUploadCmd() {
+ SwiftClientCfg cfg = mock(SwiftClientCfg.class);
+ given(cfg.getEndPoint()).willReturn("swift.endpoint");
+ given(cfg.getAccount()).willReturn("cs");
+ given(cfg.getUserName()).willReturn("sec-storage");
+ given(cfg.getKey()).willReturn("mypassword");
+ given(cfg.getStoragePolicy()).willReturn(null);
+
+ String uploadCmd = SwiftUtil.getUploadObjectCommand(cfg, "swift", "T-1", "template.vhd", 1024);
+
+ String expected = "/usr/bin/python swift -A swift.endpoint -U cs:sec-storage -K mypassword upload T-1 template.vhd";
+ assertThat(uploadCmd, is(equalTo(expected)));
+ }
+
+ @Test
+ public void testGetUploadCmdWithSegmentsBecauseOfSize() {
+ SwiftClientCfg cfg = mock(SwiftClientCfg.class);
+ given(cfg.getEndPoint()).willReturn("swift.endpoint");
+ given(cfg.getAccount()).willReturn("cs");
+ given(cfg.getUserName()).willReturn("sec-storage");
+ given(cfg.getKey()).willReturn("mypassword");
+ given(cfg.getStoragePolicy()).willReturn(null);
+
+ String uploadCmd = SwiftUtil.getUploadObjectCommand(cfg, "swift", "T-1", "template.vhd", 5368709121L);
+
+ String expected = "/usr/bin/python swift -A swift.endpoint -U cs:sec-storage -K mypassword upload T-1 template.vhd -S 5368709120";
+ assertThat(uploadCmd, is(equalTo(expected)));
+ }
+
+ @Test
+ public void testGetUploadCmdWithStoragePolicy() {
+ SwiftClientCfg cfg = mock(SwiftClientCfg.class);
+ given(cfg.getEndPoint()).willReturn("swift.endpoint");
+ given(cfg.getAccount()).willReturn("cs");
+ given(cfg.getUserName()).willReturn("sec-storage");
+ given(cfg.getKey()).willReturn("mypassword");
+ given(cfg.getStoragePolicy()).willReturn("policy1");
+
+ String uploadCmd = SwiftUtil.getUploadObjectCommand(cfg, "swift", "T-1", "template.vhd", 1024L);
+ String expected = "/usr/bin/python swift -A swift.endpoint -U cs:sec-storage -K mypassword upload T-1 template.vhd --storage-policy \"policy1\"";
+ assertThat(uploadCmd, is(equalTo(expected)));
+ }
+
+ @Test
+ public void testGetUploadCmdWithSegmentsAndStoragePolicy() {
+ SwiftClientCfg cfg = mock(SwiftClientCfg.class);
+ given(cfg.getEndPoint()).willReturn("swift.endpoint");
+ given(cfg.getAccount()).willReturn("cs");
+ given(cfg.getUserName()).willReturn("sec-storage");
+ given(cfg.getKey()).willReturn("mypassword");
+ given(cfg.getStoragePolicy()).willReturn("policy1");
+ String uploadCmd = SwiftUtil.getUploadObjectCommand(cfg, "swift", "T-1", "template.vhd", 5368709121L);
+ String expected = "/usr/bin/python swift -A swift.endpoint -U cs:sec-storage -K mypassword upload T-1 template.vhd --storage-policy \"policy1\" -S 5368709120";
+ assertThat(uploadCmd, is(equalTo(expected)));
+ }
+
+ @Test
+ public void testListContainerCmdWithStoragePolicyButNotSupportedByOperation() {
+ SwiftClientCfg cfg = mock(SwiftClientCfg.class);
+ given(cfg.getEndPoint()).willReturn("swift.endpoint");
+ given(cfg.getAccount()).willReturn("cs");
+ given(cfg.getUserName()).willReturn("sec-storage");
+ given(cfg.getKey()).willReturn("mypassword");
+ given(cfg.getStoragePolicy()).willReturn("policy1");
+
+ String uploadCmd = SwiftUtil.getSwiftContainerCmd(cfg, "swift", "list", "T-1");
+ String expected = "/usr/bin/python swift -A swift.endpoint -U cs:sec-storage -K mypassword list T-1";
+ assertThat(uploadCmd, is(equalTo(expected)));
+ }
+
+ @Test
+ public void testListContainerCmdWithoutStoragePolicy() {
+ SwiftClientCfg cfg = mock(SwiftClientCfg.class);
+ given(cfg.getEndPoint()).willReturn("swift.endpoint");
+ given(cfg.getAccount()).willReturn("cs");
+ given(cfg.getUserName()).willReturn("sec-storage");
+ given(cfg.getKey()).willReturn("mypassword");
+ given(cfg.getStoragePolicy()).willReturn(null);
+
+ String uploadCmd = SwiftUtil.getSwiftContainerCmd(cfg, "swift", "list", "T-1");
+ String expected = "/usr/bin/python swift -A swift.endpoint -U cs:sec-storage -K mypassword list T-1";
+ assertThat(uploadCmd, is(equalTo(expected)));
+ }
}
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services