You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cloudstack.apache.org by sw...@apache.org on 2016/04/06 20:20:55 UTC
[1/2] git commit: updated refs/heads/4.7 to ef115ab
Repository: cloudstack
Updated Branches:
refs/heads/4.7 4db1c0125 -> ef115ab1d
Add ability to download templates in Swift
Project: http://git-wip-us.apache.org/repos/asf/cloudstack/repo
Commit: http://git-wip-us.apache.org/repos/asf/cloudstack/commit/52fc2bab
Tree: http://git-wip-us.apache.org/repos/asf/cloudstack/tree/52fc2bab
Diff: http://git-wip-us.apache.org/repos/asf/cloudstack/diff/52fc2bab
Branch: refs/heads/4.7
Commit: 52fc2bab690d7367cf0f45cc66ed70828a0ed7d0
Parents: 6be2cc7
Author: Syed <sy...@gmail.com>
Authored: Mon Dec 14 17:32:56 2015 -0500
Committer: Syed <sy...@gmail.com>
Committed: Tue Apr 5 16:40:00 2016 -0400
----------------------------------------------------------------------
.../driver/SwiftImageStoreDriverImpl.java | 32 ++++++-
.../com/cloud/template/TemplateManagerImpl.java | 6 --
.../main/java/com/cloud/utils/SwiftUtil.java | 76 ++++++++++++++--
.../java/com/cloud/utils/SwiftUtilTest.java | 92 ++++++++++++++++++++
4 files changed, 193 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/52fc2bab/plugins/storage/image/swift/src/org/apache/cloudstack/storage/datastore/driver/SwiftImageStoreDriverImpl.java
----------------------------------------------------------------------
diff --git a/plugins/storage/image/swift/src/org/apache/cloudstack/storage/datastore/driver/SwiftImageStoreDriverImpl.java b/plugins/storage/image/swift/src/org/apache/cloudstack/storage/datastore/driver/SwiftImageStoreDriverImpl.java
index 228bbd2..2816c60 100644
--- a/plugins/storage/image/swift/src/org/apache/cloudstack/storage/datastore/driver/SwiftImageStoreDriverImpl.java
+++ b/plugins/storage/image/swift/src/org/apache/cloudstack/storage/datastore/driver/SwiftImageStoreDriverImpl.java
@@ -18,10 +18,15 @@
*/
package org.apache.cloudstack.storage.datastore.driver;
+import java.net.URL;
import java.util.Map;
+import java.util.UUID;
import javax.inject.Inject;
+import com.cloud.configuration.Config;
+import com.cloud.utils.SwiftUtil;
+import org.apache.cloudstack.framework.config.dao.ConfigurationDao;
import org.apache.log4j.Logger;
import org.apache.cloudstack.api.ApiConstants;
@@ -43,7 +48,6 @@ import com.cloud.agent.api.storage.DownloadAnswer;
import com.cloud.agent.api.to.DataObjectType;
import com.cloud.agent.api.to.DataStoreTO;
import com.cloud.agent.api.to.SwiftTO;
-import com.cloud.exception.UnsupportedServiceException;
import com.cloud.storage.Storage.ImageFormat;
import com.cloud.template.VirtualMachineTemplate;
import com.cloud.utils.exception.CloudRuntimeException;
@@ -57,6 +61,8 @@ public class SwiftImageStoreDriverImpl extends BaseImageStoreDriverImpl {
EndPointSelector _epSelector;
@Inject
StorageCacheManager cacheManager;
+ @Inject
+ ConfigurationDao _configDao;
@Override
public DataStoreTO getStoreTO(DataStore store) {
@@ -67,7 +73,29 @@ public class SwiftImageStoreDriverImpl extends BaseImageStoreDriverImpl {
@Override
public String createEntityExtractUrl(DataStore store, String installPath, ImageFormat format, DataObject dataObject) {
- throw new UnsupportedServiceException("Extract entity url is not yet supported for Swift image store provider");
+
+ SwiftTO swiftTO = (SwiftTO)store.getTO();
+ String tempKey = UUID.randomUUID().toString();
+ boolean result = SwiftUtil.setTempKey(swiftTO, tempKey);
+
+ if (!result) {
+ String errMsg = "Unable to set Temp-Key: " + tempKey;
+ s_logger.error(errMsg);
+ throw new CloudRuntimeException(errMsg);
+ }
+
+ String containerName = SwiftUtil.getContainerName(dataObject.getType().toString(), dataObject.getId());
+ String objectName = installPath.split("\\/")[1];
+ // Get extract url expiration interval set in global configuration (in seconds)
+ int urlExpirationInterval = Integer.parseInt(_configDao.getValue(Config.ExtractURLExpirationInterval.toString()));
+
+ URL swiftUrl = SwiftUtil.generateTempUrl(swiftTO, containerName, objectName, tempKey, urlExpirationInterval);
+ if (swiftUrl != null) {
+ s_logger.debug("Swift temp-url: " + swiftUrl.toString());
+ return swiftUrl.toString();
+ }
+
+ throw new CloudRuntimeException("Unable to create extraction URL");
}
@Override
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/52fc2bab/server/src/com/cloud/template/TemplateManagerImpl.java
----------------------------------------------------------------------
diff --git a/server/src/com/cloud/template/TemplateManagerImpl.java b/server/src/com/cloud/template/TemplateManagerImpl.java
index 6a8c3b6..e614142 100644
--- a/server/src/com/cloud/template/TemplateManagerImpl.java
+++ b/server/src/com/cloud/template/TemplateManagerImpl.java
@@ -1520,12 +1520,6 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager,
}
} finally {
- /*if (snapshot != null && snapshot.getSwiftId() != null
- && secondaryStorageURL != null && zoneId != null
- && accountId != null && volumeId != null) {
- _snapshotMgr.deleteSnapshotsForVolume(secondaryStorageURL,
- zoneId, accountId, volumeId);
- }*/
if (privateTemplate == null) {
final VolumeVO volumeFinal = volume;
final SnapshotVO snapshotFinal = snapshot;
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/52fc2bab/utils/src/main/java/com/cloud/utils/SwiftUtil.java
----------------------------------------------------------------------
diff --git a/utils/src/main/java/com/cloud/utils/SwiftUtil.java b/utils/src/main/java/com/cloud/utils/SwiftUtil.java
index 1136818..ce1bee3 100644
--- a/utils/src/main/java/com/cloud/utils/SwiftUtil.java
+++ b/utils/src/main/java/com/cloud/utils/SwiftUtil.java
@@ -20,8 +20,14 @@
package com.cloud.utils;
import java.io.File;
-import java.util.Arrays;
+import java.net.URL;
+import java.security.InvalidKeyException;
+import java.security.NoSuchAlgorithmException;
+import java.security.SignatureException;
import java.util.Map;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Formatter;
import org.apache.log4j.Logger;
@@ -29,9 +35,15 @@ import com.cloud.utils.exception.CloudRuntimeException;
import com.cloud.utils.script.OutputInterpreter;
import com.cloud.utils.script.Script;
+import javax.crypto.Mac;
+import javax.crypto.spec.SecretKeySpec;
+
public class SwiftUtil {
private static Logger logger = Logger.getLogger(SwiftUtil.class);
private static final long SWIFT_MAX_SIZE = 5L * 1024L * 1024L * 1024L;
+ private static final String HMAC_SHA1_ALGORITHM = "HmacSHA1";
+
+
public interface SwiftClientCfg {
String getAccount();
@@ -143,8 +155,7 @@ public class SwiftUtil {
OutputInterpreter.AllLinesParser parser = new OutputInterpreter.AllLinesParser();
String result = command.execute(parser);
if (result == null && parser.getLines() != null && !parser.getLines().equalsIgnoreCase("")) {
- String[] lines = parser.getLines().split("\\n");
- return lines;
+ return parser.getLines().split("\\n");
} else {
if (result != null) {
String errMsg = "swiftList failed , err=" + result;
@@ -161,7 +172,7 @@ public class SwiftUtil {
int firstIndexOfSeparator = swiftPath.indexOf(File.separator);
String container = swiftPath.substring(0, firstIndexOfSeparator);
String srcPath = swiftPath.substring(firstIndexOfSeparator + 1);
- String destFilePath = null;
+ String destFilePath;
if (destDirectory.isDirectory()) {
destFilePath = destDirectory.getAbsolutePath() + File.separator + srcPath;
} else {
@@ -171,7 +182,7 @@ public class SwiftUtil {
Script command = new Script("/bin/bash", logger);
command.add("-c");
command.add("/usr/bin/python " + swiftCli + " -A " + cfg.getEndPoint() + " -U " + cfg.getAccount() + ":" + cfg.getUserName() + " -K " + cfg.getKey() +
- " download " + container + " " + srcPath + " -o " + destFilePath);
+ " download " + container + " " + srcPath + " -o " + destFilePath);
OutputInterpreter.AllLinesParser parser = new OutputInterpreter.AllLinesParser();
String result = command.execute(parser);
if (result != null) {
@@ -236,4 +247,59 @@ public class SwiftUtil {
command.execute(parser);
return true;
}
+
+ public static boolean setTempKey(SwiftClientCfg cfg, String tempKey){
+
+ Map<String, String> tempKeyMap = new HashMap<>();
+ tempKeyMap.put("Temp-URL-Key", tempKey);
+ return postMeta(cfg, "", "", tempKeyMap);
+
+ }
+
+ public static URL generateTempUrl(SwiftClientCfg cfg, String container, String object, String tempKey, int urlExpirationInterval) {
+
+ int currentTime = (int) (System.currentTimeMillis() / 1000L);
+ int expirationSeconds = currentTime + urlExpirationInterval;
+
+ try {
+
+ URL endpoint = new URL(cfg.getEndPoint());
+ String method = "GET";
+ String path = String.format("/v1/AUTH_%s/%s/%s", cfg.getAccount(), container, object);
+
+ //sign the request
+ String hmacBody = String.format("%s\n%d\n%s", method, expirationSeconds, path);
+ String signature = calculateRFC2104HMAC(hmacBody, tempKey);
+ path += String.format("?temp_url_sig=%s&temp_url_expires=%d", signature, expirationSeconds);
+
+ //generate the temp url
+ URL tempUrl = new URL(endpoint.getProtocol(), endpoint.getHost(), endpoint.getPort(), path);
+
+ return tempUrl;
+
+ } catch (Exception e) {
+ logger.error(e.getMessage());
+ throw new CloudRuntimeException(e.getMessage());
+ }
+
+ }
+
+ public static String calculateRFC2104HMAC(String data, String key)
+ throws SignatureException, NoSuchAlgorithmException, InvalidKeyException {
+
+ SecretKeySpec signingKey = new SecretKeySpec(key.getBytes(), HMAC_SHA1_ALGORITHM);
+ Mac mac = Mac.getInstance(HMAC_SHA1_ALGORITHM);
+ mac.init(signingKey);
+ return toHexString(mac.doFinal(data.getBytes()));
+
+ }
+
+ public static String toHexString(byte[] bytes) {
+
+ Formatter formatter = new Formatter();
+ for (byte b : bytes) {
+ formatter.format("%02x", b);
+ }
+ return formatter.toString();
+ }
}
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/52fc2bab/utils/src/test/java/com/cloud/utils/SwiftUtilTest.java
----------------------------------------------------------------------
diff --git a/utils/src/test/java/com/cloud/utils/SwiftUtilTest.java b/utils/src/test/java/com/cloud/utils/SwiftUtilTest.java
new file mode 100644
index 0000000..55f7a34
--- /dev/null
+++ b/utils/src/test/java/com/cloud/utils/SwiftUtilTest.java
@@ -0,0 +1,92 @@
+//
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+//
+
+package com.cloud.utils;
+
+import org.junit.Test;
+import org.mockito.Mockito;
+
+import java.net.URL;
+import java.security.InvalidKeyException;
+import java.security.NoSuchAlgorithmException;
+import java.security.SignatureException;
+
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.Mockito.when;
+
+
+public class SwiftUtilTest {
+
+ @Test
+ public void testCalculateRFC2104HMAC() throws NoSuchAlgorithmException, SignatureException, InvalidKeyException {
+ String inputData = "testData";
+ String inputKey = "testKey";
+ String expected = "1d541ecb5cdb2d850716bfd55585e20a1cd8984b";
+ String output = SwiftUtil.calculateRFC2104HMAC(inputData, inputKey);
+
+ assertEquals(expected, output);
+ }
+
+ @Test
+ public void testToHexString(){
+ final byte[] input = "testing".getBytes();
+ final String expected = "74657374696e67";
+ final String result = SwiftUtil.toHexString(input);
+ assertEquals(expected, result);
+ }
+
+ @Test
+ public void testGenerateTempUrl() {
+
+ SwiftUtil.SwiftClientCfg cfg = Mockito.mock(SwiftUtil.SwiftClientCfg.class);
+ when(cfg.getEndPoint()).thenReturn("http://localhost:8080/v1/");
+ when(cfg.getAccount()).thenReturn("test");
+
+ String container = "testContainer";
+ String object = "testObject";
+ String tempKey = "testKey";
+ int urlExpirationInterval = 3600;
+ String expected = "http://localhost:8080/v1/AUTH_test/testContainer/testObject";
+ URL output = SwiftUtil.generateTempUrl(cfg, container, object, tempKey, urlExpirationInterval);
+
+ assertTrue(output.toString().contains(expected));
+ }
+
+ @Test
+ public void testSplitSwiftPath(){
+ String input = "container/object";
+ String[] output = SwiftUtil.splitSwiftPath(input);
+ String[] expected = {"container", "object"};
+
+ assertArrayEquals(expected, output);
+ }
+
+ @Test
+ public void testGetContainerName(){
+
+ String inputType = "Template";
+ long inputId = 45;
+ String output = SwiftUtil.getContainerName(inputType, inputId);
+ String expected = "T-45";
+
+ assertEquals(expected, output);
+ }
+}
[2/2] git commit: updated refs/heads/4.7 to ef115ab
Posted by sw...@apache.org.
Merge pull request #1332 from syed/swift-download
Add ability to download templates in SwiftThis PR adds the ability to download templates when using Swift as a secondary storage. Uses the "temp_url" feature of Swift so that tempates can be downloaded without authenticaiton.
* pr/1332:
Add ability to download templates in Swift
Signed-off-by: Will Stevens <wi...@gmail.com>
Project: http://git-wip-us.apache.org/repos/asf/cloudstack/repo
Commit: http://git-wip-us.apache.org/repos/asf/cloudstack/commit/ef115ab1
Tree: http://git-wip-us.apache.org/repos/asf/cloudstack/tree/ef115ab1
Diff: http://git-wip-us.apache.org/repos/asf/cloudstack/diff/ef115ab1
Branch: refs/heads/4.7
Commit: ef115ab1d0fd53ef083bf4287e4036a995dfeaaa
Parents: 4db1c01 52fc2ba
Author: Will Stevens <wi...@gmail.com>
Authored: Wed Apr 6 14:19:44 2016 -0400
Committer: Will Stevens <wi...@gmail.com>
Committed: Wed Apr 6 14:19:44 2016 -0400
----------------------------------------------------------------------
.../driver/SwiftImageStoreDriverImpl.java | 32 ++++++-
.../com/cloud/template/TemplateManagerImpl.java | 6 --
.../main/java/com/cloud/utils/SwiftUtil.java | 76 ++++++++++++++--
.../java/com/cloud/utils/SwiftUtilTest.java | 92 ++++++++++++++++++++
4 files changed, 193 insertions(+), 13 deletions(-)
----------------------------------------------------------------------