You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cloudstack.apache.org by da...@apache.org on 2016/05/24 09:49:25 UTC
[20/48] git commit: updated refs/heads/4.9-bountycastle-daan to
98bf0ca
Fix Sync of template.properties in Swift
Project: http://git-wip-us.apache.org/repos/asf/cloudstack/repo
Commit: http://git-wip-us.apache.org/repos/asf/cloudstack/commit/f5ac8ddd
Tree: http://git-wip-us.apache.org/repos/asf/cloudstack/tree/f5ac8ddd
Diff: http://git-wip-us.apache.org/repos/asf/cloudstack/diff/f5ac8ddd
Branch: refs/heads/4.9-bountycastle-daan
Commit: f5ac8dddedaec6ed5e21069f150f86229b43f02c
Parents: 7a0b37a
Author: Syed <sy...@gmail.com>
Authored: Mon Dec 14 17:37:28 2015 -0500
Committer: Syed <sy...@gmail.com>
Committed: Fri May 13 22:08:16 2016 +0000
----------------------------------------------------------------------
.../datastore/ObjectInDataStoreManagerImpl.java | 1 +
server/src/com/cloud/test/TestAppender.java | 178 +++++++++
services/secondary-storage/server/pom.xml | 3 -
.../resource/NfsSecondaryStorageResource.java | 373 ++++++++++++-------
.../LocalNfsSecondaryStorageResourceTest.java | 44 ++-
.../NfsSecondaryStorageResourceTest.java | 118 +++---
6 files changed, 481 insertions(+), 236 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/f5ac8ddd/engine/storage/src/org/apache/cloudstack/storage/datastore/ObjectInDataStoreManagerImpl.java
----------------------------------------------------------------------
diff --git a/engine/storage/src/org/apache/cloudstack/storage/datastore/ObjectInDataStoreManagerImpl.java b/engine/storage/src/org/apache/cloudstack/storage/datastore/ObjectInDataStoreManagerImpl.java
index b1d6127..5117b7c 100644
--- a/engine/storage/src/org/apache/cloudstack/storage/datastore/ObjectInDataStoreManagerImpl.java
+++ b/engine/storage/src/org/apache/cloudstack/storage/datastore/ObjectInDataStoreManagerImpl.java
@@ -156,6 +156,7 @@ public class ObjectInDataStoreManagerImpl implements ObjectInDataStoreManager {
// template.properties
// there
}
+
ts.setInstallPath(installPath);
ts.setState(ObjectInDataStoreStateMachine.State.Allocated);
ts = templateDataStoreDao.persist(ts);
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/f5ac8ddd/server/src/com/cloud/test/TestAppender.java
----------------------------------------------------------------------
diff --git a/server/src/com/cloud/test/TestAppender.java b/server/src/com/cloud/test/TestAppender.java
new file mode 100644
index 0000000..539cac1
--- /dev/null
+++ b/server/src/com/cloud/test/TestAppender.java
@@ -0,0 +1,178 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing,
+* software distributed under the License is distributed on an
+* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+* KIND, either express or implied. See the License for the
+* specific language governing permissions and limitations
+* under the License.
+*/
+package com.cloud.test;
+
+import com.google.common.base.Joiner;
+import com.google.common.base.Objects;
+import com.google.common.collect.ImmutableMap;
+import org.apache.log4j.AppenderSkeleton;
+import org.apache.log4j.Level;
+import org.apache.log4j.Logger;
+import org.apache.log4j.spi.LoggingEvent;
+import org.springframework.util.Assert;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.regex.Pattern;
+
+import static com.google.common.base.Preconditions.checkArgument;
+import static com.google.common.base.Preconditions.checkState;
+import static com.google.common.base.Strings.isNullOrEmpty;
+import static java.lang.String.format;
+import static org.apache.log4j.Level.ALL;
+import static org.apache.log4j.Level.DEBUG;
+import static org.apache.log4j.Level.ERROR;
+import static org.apache.log4j.Level.FATAL;
+import static org.apache.log4j.Level.INFO;
+import static org.apache.log4j.Level.OFF;
+
+/**
+*
+* Tracks one or more patterns to determine whether or not they have been
+* logged. It uses a streaming approach to determine whether or not a message
+* has a occurred to prevent unnecessary memory consumption. Instances of this
+* of this class are created using the {@link TestAppenderBuilder}.
+*
+* To use this class, register a one or more expected patterns by level as part
+* of the test setup and retain an reference to the appender instance. After the
+* expected logging events have occurred in the test case, call
+* {@link TestAppender#assertMessagesLogged()} which will fail the test if any of the
+* expected patterns were not logged.
+*
+*/
+public final class TestAppender extends AppenderSkeleton {
+ private final static String APPENDER_NAME = "test_appender";
+ private final ImmutableMap<Level, Set<PatternResult>> expectedPatternResults;
+ private TestAppender(final Map<Level, Set<PatternResult>> expectedPatterns) {
+ super();
+ expectedPatternResults = ImmutableMap.copyOf(expectedPatterns);
+ }
+ protected void append(LoggingEvent loggingEvent) {
+ checkArgument(loggingEvent != null, "append requires a non-null loggingEvent");
+ final Level level = loggingEvent.getLevel();
+ checkState(expectedPatternResults.containsKey(level), "level " + level + " not supported by append");
+ for (final PatternResult patternResult : expectedPatternResults.get(level)) {
+ if (patternResult.getPattern().matcher(loggingEvent.getRenderedMessage()).matches()) {
+ patternResult.markFound();
+ }
+ }
+ }
+
+ public void close() {
+// Do nothing ...
+ }
+ public boolean requiresLayout() {
+ return false;
+ }
+ public void assertMessagesLogged() {
+ final List<String> unloggedPatterns = new ArrayList<>();
+ for (final Map.Entry<Level, Set<PatternResult>> expectedPatternResult : expectedPatternResults.entrySet()) {
+ for (final PatternResult patternResults : expectedPatternResult.getValue()) {
+ if (!patternResults.isFound()) {
+ unloggedPatterns.add(format("%1$s was not logged for level %2$s",
+ patternResults.getPattern().toString(), expectedPatternResult.getKey()));
+ }
+ }
+ }
+ if (!unloggedPatterns.isEmpty()) {
+ //Raise an assert
+ Assert.isTrue(false, Joiner.on(",").join(unloggedPatterns));
+ }
+ }
+
+ private static final class PatternResult {
+ private final Pattern pattern;
+ private boolean foundFlag = false;
+ private PatternResult(Pattern pattern) {
+ super();
+ this.pattern = pattern;
+ }
+ public Pattern getPattern() {
+ return pattern;
+ }
+ public void markFound() {
+ // This operation is thread-safe because the value will only ever be switched from false to true. Therefore,
+ // multiple threads mutating the value for a pattern will not corrupt the value ...
+ foundFlag = true;
+ }
+ public boolean isFound() {
+ return foundFlag;
+ }
+ @Override
+ public boolean equals(Object thatObject) {
+ if (this == thatObject) {
+ return true;
+ }
+ if (thatObject == null || getClass() != thatObject.getClass()) {
+ return false;
+ }
+ PatternResult thatPatternResult = (PatternResult) thatObject;
+ return foundFlag == thatPatternResult.foundFlag &&
+ Objects.equal(pattern, thatPatternResult.pattern);
+ }
+ @Override
+ public int hashCode() {
+ return Objects.hashCode(pattern, foundFlag);
+ }
+ @Override
+ public String toString() {
+ return format("Pattern Result [ pattern: %1$s, markFound: %2$s ]", pattern.toString(), foundFlag);
+ }
+ }
+
+ public static final class TestAppenderBuilder {
+ private final Map<Level, Set<PatternResult>> expectedPatterns;
+ public TestAppenderBuilder() {
+ super();
+ expectedPatterns = new HashMap<>();
+ expectedPatterns.put(ALL, new HashSet<PatternResult>());
+ expectedPatterns.put(DEBUG, new HashSet<PatternResult>());
+ expectedPatterns.put(ERROR, new HashSet<PatternResult>());
+ expectedPatterns.put(FATAL, new HashSet<PatternResult>());
+ expectedPatterns.put(INFO, new HashSet<PatternResult>());
+ expectedPatterns.put(OFF, new HashSet<PatternResult>());
+ }
+ public TestAppenderBuilder addExpectedPattern(final Level level, final String pattern) {
+ checkArgument(level != null, "addExpectedPattern requires a non-null level");
+ checkArgument(!isNullOrEmpty(pattern), "addExpectedPattern requires a non-blank pattern");
+ checkState(expectedPatterns.containsKey(level), "level " + level + " is not supported by " + getClass().getName());
+ expectedPatterns.get(level).add(new PatternResult(Pattern.compile(pattern)));
+ return this;
+ }
+ public TestAppender build() {
+ return new TestAppender(expectedPatterns);
+ }
+ }
+ /**
+ *
+ * Attaches a {@link TestAppender} to a {@link Logger} and ensures that it is the only
+ * test appender attached to the logger.
+ *
+ * @param logger The logger which will be monitored by the test
+ * @param testAppender The test appender to attach to {@code logger}
+ */
+ public static void safeAddAppender(Logger logger, TestAppender testAppender) {
+ logger.removeAppender(APPENDER_NAME);
+ logger.addAppender(testAppender);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/f5ac8ddd/services/secondary-storage/server/pom.xml
----------------------------------------------------------------------
diff --git a/services/secondary-storage/server/pom.xml b/services/secondary-storage/server/pom.xml
index 417e0a6..63aef27 100644
--- a/services/secondary-storage/server/pom.xml
+++ b/services/secondary-storage/server/pom.xml
@@ -26,9 +26,6 @@
<version>4.7.2-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
- <properties>
- <skipTests>true</skipTests>
- </properties>
<dependencies>
<dependency>
<groupId>log4j</groupId>
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/f5ac8ddd/services/secondary-storage/server/src/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java
----------------------------------------------------------------------
diff --git a/services/secondary-storage/server/src/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java b/services/secondary-storage/server/src/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java
index 768a177..f512da0 100644
--- a/services/secondary-storage/server/src/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java
+++ b/services/secondary-storage/server/src/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java
@@ -16,91 +16,7 @@
// under the License.
package org.apache.cloudstack.storage.resource;
-import static com.cloud.utils.storage.S3.S3Utils.putFile;
-import static com.cloud.utils.StringUtils.join;
-import static java.lang.String.format;
-import static java.util.Arrays.asList;
-import static org.apache.commons.lang.StringUtils.substringAfterLast;
-
-import java.io.BufferedReader;
-import java.io.BufferedWriter;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileOutputStream;
-import java.io.FileReader;
-import java.io.FileWriter;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.UnsupportedEncodingException;
-import java.math.BigInteger;
-import java.net.InetAddress;
-import java.net.URI;
-import java.net.UnknownHostException;
-import java.security.MessageDigest;
-import java.security.NoSuchAlgorithmException;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.UUID;
-
-import javax.naming.ConfigurationException;
-
-import com.cloud.exception.InvalidParameterValueException;
-import com.cloud.storage.Storage;
-import com.cloud.storage.template.TemplateConstants;
-import com.cloud.utils.EncryptionUtil;
-import com.google.gson.Gson;
-import com.google.gson.GsonBuilder;
-import io.netty.bootstrap.ServerBootstrap;
-import io.netty.channel.Channel;
-import io.netty.channel.ChannelInitializer;
-import io.netty.channel.ChannelPipeline;
-import io.netty.channel.EventLoopGroup;
-import io.netty.channel.nio.NioEventLoopGroup;
-import io.netty.channel.socket.SocketChannel;
-import io.netty.channel.socket.nio.NioServerSocketChannel;
-import io.netty.handler.codec.http.HttpContentCompressor;
-import io.netty.handler.codec.http.HttpRequestDecoder;
-import io.netty.handler.codec.http.HttpResponseEncoder;
-import io.netty.handler.logging.LogLevel;
-import io.netty.handler.logging.LoggingHandler;
-import org.apache.cloudstack.storage.command.TemplateOrVolumePostUploadCommand;
-import org.apache.cloudstack.storage.template.UploadEntity;
-import org.apache.cloudstack.utils.imagestore.ImageStoreUtil;
-import org.apache.commons.codec.digest.DigestUtils;
-import org.apache.commons.io.FileUtils;
-import org.apache.commons.io.FilenameUtils;
-import org.apache.commons.lang.StringUtils;
-import org.apache.http.HttpEntity;
-import org.apache.http.HttpResponse;
-import org.apache.http.NameValuePair;
-import org.apache.http.client.HttpClient;
-import org.apache.http.client.methods.HttpGet;
-import org.apache.http.client.utils.URLEncodedUtils;
-import org.apache.http.impl.client.DefaultHttpClient;
-import org.apache.log4j.Logger;
-
import com.amazonaws.services.s3.model.S3ObjectSummary;
-
-import org.apache.cloudstack.framework.security.keystore.KeystoreManager;
-import org.apache.cloudstack.storage.command.CopyCmdAnswer;
-import org.apache.cloudstack.storage.command.CopyCommand;
-import org.apache.cloudstack.storage.command.DeleteCommand;
-import org.apache.cloudstack.storage.command.DownloadCommand;
-import org.apache.cloudstack.storage.command.DownloadProgressCommand;
-import org.apache.cloudstack.storage.command.UploadStatusAnswer;
-import org.apache.cloudstack.storage.command.UploadStatusAnswer.UploadStatus;
-import org.apache.cloudstack.storage.command.UploadStatusCommand;
-import org.apache.cloudstack.storage.template.DownloadManager;
-import org.apache.cloudstack.storage.template.DownloadManagerImpl;
-import org.apache.cloudstack.storage.template.DownloadManagerImpl.ZfsPathParser;
-import org.apache.cloudstack.storage.template.UploadManager;
-import org.apache.cloudstack.storage.template.UploadManagerImpl;
-import org.apache.cloudstack.storage.to.SnapshotObjectTO;
-import org.apache.cloudstack.storage.to.TemplateObjectTO;
-import org.apache.cloudstack.storage.to.VolumeObjectTO;
-
import com.cloud.agent.api.Answer;
import com.cloud.agent.api.CheckHealthAnswer;
import com.cloud.agent.api.CheckHealthCommand;
@@ -135,11 +51,13 @@ import com.cloud.agent.api.to.NfsTO;
import com.cloud.agent.api.to.S3TO;
import com.cloud.agent.api.to.SwiftTO;
import com.cloud.exception.InternalErrorException;
+import com.cloud.exception.InvalidParameterValueException;
import com.cloud.host.Host;
import com.cloud.host.Host.Type;
import com.cloud.hypervisor.Hypervisor.HypervisorType;
import com.cloud.resource.ServerResourceBase;
import com.cloud.storage.DataStoreRole;
+import com.cloud.storage.Storage;
import com.cloud.storage.Storage.ImageFormat;
import com.cloud.storage.StorageLayer;
import com.cloud.storage.VMTemplateStorageResourceAssoc;
@@ -149,24 +67,102 @@ import com.cloud.storage.template.Processor.FormatInfo;
import com.cloud.storage.template.QCOW2Processor;
import com.cloud.storage.template.RawImageProcessor;
import com.cloud.storage.template.TARProcessor;
+import com.cloud.storage.template.TemplateConstants;
import com.cloud.storage.template.TemplateLocation;
import com.cloud.storage.template.TemplateProp;
import com.cloud.storage.template.VhdProcessor;
import com.cloud.storage.template.VmdkProcessor;
+import com.cloud.utils.EncryptionUtil;
import com.cloud.utils.NumbersUtil;
-import com.cloud.utils.storage.S3.S3Utils;
import com.cloud.utils.SwiftUtil;
import com.cloud.utils.exception.CloudRuntimeException;
import com.cloud.utils.net.NetUtils;
import com.cloud.utils.script.OutputInterpreter;
import com.cloud.utils.script.Script;
+import com.cloud.utils.storage.S3.S3Utils;
import com.cloud.vm.SecondaryStorageVm;
+import com.google.gson.Gson;
+import com.google.gson.GsonBuilder;
+import io.netty.bootstrap.ServerBootstrap;
+import io.netty.channel.Channel;
+import io.netty.channel.ChannelInitializer;
+import io.netty.channel.ChannelPipeline;
+import io.netty.channel.EventLoopGroup;
+import io.netty.channel.nio.NioEventLoopGroup;
+import io.netty.channel.socket.SocketChannel;
+import io.netty.channel.socket.nio.NioServerSocketChannel;
+import io.netty.handler.codec.http.HttpContentCompressor;
+import io.netty.handler.codec.http.HttpRequestDecoder;
+import io.netty.handler.codec.http.HttpResponseEncoder;
+import io.netty.handler.logging.LogLevel;
+import io.netty.handler.logging.LoggingHandler;
+import org.apache.cloudstack.framework.security.keystore.KeystoreManager;
+import org.apache.cloudstack.storage.command.CopyCmdAnswer;
+import org.apache.cloudstack.storage.command.CopyCommand;
+import org.apache.cloudstack.storage.command.DeleteCommand;
+import org.apache.cloudstack.storage.command.DownloadCommand;
+import org.apache.cloudstack.storage.command.DownloadProgressCommand;
+import org.apache.cloudstack.storage.command.TemplateOrVolumePostUploadCommand;
+import org.apache.cloudstack.storage.command.UploadStatusAnswer;
+import org.apache.cloudstack.storage.command.UploadStatusAnswer.UploadStatus;
+import org.apache.cloudstack.storage.command.UploadStatusCommand;
+import org.apache.cloudstack.storage.template.DownloadManager;
+import org.apache.cloudstack.storage.template.DownloadManagerImpl;
+import org.apache.cloudstack.storage.template.DownloadManagerImpl.ZfsPathParser;
+import org.apache.cloudstack.storage.template.UploadEntity;
+import org.apache.cloudstack.storage.template.UploadManager;
+import org.apache.cloudstack.storage.template.UploadManagerImpl;
+import org.apache.cloudstack.storage.to.SnapshotObjectTO;
+import org.apache.cloudstack.storage.to.TemplateObjectTO;
+import org.apache.cloudstack.storage.to.VolumeObjectTO;
+import org.apache.cloudstack.utils.imagestore.ImageStoreUtil;
+import org.apache.commons.codec.digest.DigestUtils;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.FilenameUtils;
+import org.apache.commons.lang.StringUtils;
+import org.apache.http.HttpEntity;
+import org.apache.http.HttpResponse;
+import org.apache.http.NameValuePair;
+import org.apache.http.client.HttpClient;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.client.utils.URLEncodedUtils;
+import org.apache.http.impl.client.DefaultHttpClient;
+import org.apache.log4j.Logger;
import org.joda.time.DateTime;
import org.joda.time.format.ISODateTimeFormat;
+import javax.naming.ConfigurationException;
+import java.io.BufferedReader;
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.FileReader;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.UnsupportedEncodingException;
+import java.math.BigInteger;
+import java.net.InetAddress;
+import java.net.URI;
+import java.net.UnknownHostException;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+
+import static com.cloud.utils.StringUtils.join;
+import static com.cloud.utils.storage.S3.S3Utils.putFile;
+import static java.lang.String.format;
+import static java.util.Arrays.asList;
+import static org.apache.commons.lang.StringUtils.substringAfterLast;
+
public class NfsSecondaryStorageResource extends ServerResourceBase implements SecondaryStorageResource {
- private static final Logger s_logger = Logger.getLogger(NfsSecondaryStorageResource.class);
+ public static final Logger s_logger = Logger.getLogger(NfsSecondaryStorageResource.class);
private static final String TEMPLATE_ROOT_DIR = "template/tmpl";
private static final String VOLUME_ROOT_DIR = "volumes";
@@ -499,10 +495,10 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S
String destFileFullPath = destFile.getAbsolutePath() + File.separator + fileName;
s_logger.debug("copy snapshot " + srcFile.getAbsolutePath() + " to template " + destFileFullPath);
Script.runSimpleBashScript("cp " + srcFile.getAbsolutePath() + " " + destFileFullPath);
- String metaFileName = destFile.getAbsolutePath() + File.separator + "template.properties";
+ String metaFileName = destFile.getAbsolutePath() + File.separator + _tmpltpp;
File metaFile = new File(metaFileName);
try {
- _storage.create(destFile.getAbsolutePath(), "template.properties");
+ _storage.create(destFile.getAbsolutePath(), _tmpltpp);
try ( // generate template.properties file
FileWriter writer = new FileWriter(metaFile);
BufferedWriter bufferWriter = new BufferedWriter(writer);
@@ -597,32 +593,14 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S
return answer;
}
s_logger.debug("starting copy template to swift");
- DataTO newTemplate = answer.getNewData();
- File templateFile = getFile(newTemplate.getPath(), ((NfsTO)srcDataStore).getUrl());
- SwiftTO swift = (SwiftTO)destDataStore;
- String containterName = SwiftUtil.getContainerName(destData.getObjectType().toString(), destData.getId());
- String swiftPath = SwiftUtil.putObject(swift, templateFile, containterName, templateFile.getName());
- //upload template.properties
- File properties = new File(templateFile.getParent() + File.separator + _tmpltpp);
- if (properties.exists()) {
- SwiftUtil.putObject(swift, properties, containterName, _tmpltpp);
- }
+ TemplateObjectTO newTemplate = (TemplateObjectTO)answer.getNewData();
+ newTemplate.setDataStore(srcDataStore);
+ CopyCommand newCpyCmd = new CopyCommand(newTemplate, destData, cmd.getWait(), cmd.executeInSequence());
+ Answer result = copyFromNfsToSwift(newCpyCmd);
- //clean up template data on staging area
- try {
- DeleteCommand deleteCommand = new DeleteCommand(newTemplate);
- execute(deleteCommand);
- } catch (Exception e) {
- s_logger.debug("Failed to clean up staging area:", e);
- }
+ cleanupStagingNfs(newTemplate);
+ return result;
- TemplateObjectTO template = new TemplateObjectTO();
- template.setPath(swiftPath);
- template.setSize(templateFile.length());
- template.setPhysicalSize(template.getSize());
- SnapshotObjectTO snapshot = (SnapshotObjectTO)srcData;
- template.setFormat(snapshot.getVolume().getFormat());
- return new CopyCmdAnswer(template);
} else if (destDataStore instanceof S3TO) {
//create template on the same data store
CopyCmdAnswer answer =
@@ -635,18 +613,27 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S
newTemplate.setDataStore(srcDataStore);
CopyCommand newCpyCmd = new CopyCommand(newTemplate, destData, cmd.getWait(), cmd.executeInSequence());
Answer result = copyFromNfsToS3(newCpyCmd);
- //clean up template data on staging area
- try {
- DeleteCommand deleteCommand = new DeleteCommand(newTemplate);
- execute(deleteCommand);
- } catch (Exception e) {
- s_logger.debug("Failed to clean up staging area:", e);
- }
+
+ cleanupStagingNfs(newTemplate);
+
return result;
}
}
- s_logger.debug("Failed to create templat from snapshot");
- return new CopyCmdAnswer("Unsupported prototcol");
+ s_logger.debug("Failed to create template from snapshot");
+ return new CopyCmdAnswer("Unsupported protocol");
+ }
+
+ /**
+ * clean up template data on staging area
+ * @param newTemplate: The template on the secondary storage that needs to be cleaned up
+ */
+ protected void cleanupStagingNfs(TemplateObjectTO newTemplate) {
+ try {
+ DeleteCommand deleteCommand = new DeleteCommand(newTemplate);
+ execute(deleteCommand);
+ } catch (Exception e) {
+ s_logger.debug("Failed to clean up staging area:", e);
+ }
}
protected Answer copyFromNfsToImage(CopyCommand cmd) {
@@ -759,22 +746,18 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S
String container = "T-" + cmd.getId();
String swiftPath = SwiftUtil.putObject(swiftTO, file, container, null);
+ long virtualSize = getVirtualSize(file, getTemplateFormat(file.getName()));
+ long size = file.length();
+ String uniqueName = cmd.getName();
+
//put metda file
File uniqDir = _storage.createUniqDir();
- String metaFileName = uniqDir.getAbsolutePath() + File.separator + "template.properties";
- _storage.create(uniqDir.getAbsolutePath(), "template.properties");
- File metaFile = new File(metaFileName);
- FileWriter writer = new FileWriter(metaFile);
- BufferedWriter bufferWriter = new BufferedWriter(writer);
- bufferWriter.write("uniquename=" + cmd.getName());
- bufferWriter.write("\n");
- bufferWriter.write("filename=" + fileName);
- bufferWriter.write("\n");
- bufferWriter.write("size=" + file.length());
- bufferWriter.close();
- writer.close();
-
- SwiftUtil.putObject(swiftTO, metaFile, container, "template.properties");
+ String metaFileName = uniqDir.getAbsolutePath() + File.separator + _tmpltpp;
+ _storage.create(uniqDir.getAbsolutePath(), _tmpltpp);
+
+ File metaFile = swiftWriteMetadataFile(metaFileName, uniqueName, fileName, size, virtualSize);
+
+ SwiftUtil.putObject(swiftTO, metaFile, container, _tmpltpp);
metaFile.delete();
uniqDir.delete();
String md5sum = null;
@@ -785,7 +768,7 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S
}
DownloadAnswer answer =
- new DownloadAnswer(null, 100, null, VMTemplateStorageResourceAssoc.Status.DOWNLOADED, swiftPath, swiftPath, file.length(), file.length(), md5sum);
+ new DownloadAnswer(null, 100, null, VMTemplateStorageResourceAssoc.Status.DOWNLOADED, swiftPath, swiftPath, virtualSize, file.length(), md5sum);
return answer;
} catch (IOException e) {
s_logger.debug("Failed to register template into swift", e);
@@ -942,6 +925,118 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S
}
}
+ /***
+ *This method will create a file using the filenName and metaFileName.
+ *That file will contain the given attributes (unique name, file name, size, and virtualSize).
+ *
+ * @param metaFileName : The path of the metadata file
+ * @param filename :attribute: Filename of the template
+ * @param uniqueName :attribute: Unique name of the template
+ * @param size :attribute: physical size of the template
+ * @param virtualSize :attribute: virtual size of the template
+ * @return File representing the metadata file
+ * @throws IOException
+ */
+
+ protected File swiftWriteMetadataFile(String metaFileName, String uniqueName, String filename, long size, long virtualSize) throws IOException {
+ File metaFile = new File(metaFileName);
+ FileWriter writer = new FileWriter(metaFile);
+ BufferedWriter bufferWriter = new BufferedWriter(writer);
+ bufferWriter.write("uniquename=" + uniqueName);
+ bufferWriter.write("\n");
+ bufferWriter.write("filename=" + filename);
+ bufferWriter.write("\n");
+ bufferWriter.write("size=" + size);
+ bufferWriter.write("\n");
+ bufferWriter.write("virtualsize=" + virtualSize);
+ bufferWriter.close();
+ writer.close();
+ return metaFile;
+ }
+
+ /**
+ * Creates a template.properties for Swift with its correct unique name
+ *
+ * @param swift The swift object
+ * @param srcFile Source file on the staging NFS
+ * @param containerName Destination container
+ * @return true on successful write
+ */
+ protected boolean swiftUploadMetadataFile(SwiftTO swift, File srcFile, String containerName) throws IOException {
+
+ String uniqueName = FilenameUtils.getBaseName(srcFile.getName());
+
+ File uniqDir = _storage.createUniqDir();
+ String metaFileName = uniqDir.getAbsolutePath() + File.separator + _tmpltpp;
+ _storage.create(uniqDir.getAbsolutePath(), _tmpltpp);
+
+ long virtualSize = getVirtualSize(srcFile, getTemplateFormat(srcFile.getName()));
+
+ File metaFile = swiftWriteMetadataFile(metaFileName,
+ uniqueName,
+ srcFile.getName(),
+ srcFile.length(),
+ virtualSize);
+
+ SwiftUtil.putObject(swift, metaFile, containerName, _tmpltpp);
+ metaFile.delete();
+ uniqDir.delete();
+
+ return true;
+ }
+
+ /**
+ * Copies data from NFS and uploads it into a Swift container
+ *
+ * @param cmd CopyComand
+ * @return CopyCmdAnswer
+ */
+ protected Answer copyFromNfsToSwift(CopyCommand cmd) {
+
+ final DataTO srcData = cmd.getSrcTO();
+ final DataTO destData = cmd.getDestTO();
+
+ DataStoreTO srcDataStore = srcData.getDataStore();
+ NfsTO srcStore = (NfsTO)srcDataStore;
+ DataStoreTO destDataStore = destData.getDataStore();
+ File srcFile = getFile(srcData.getPath(), srcStore.getUrl());
+
+ SwiftTO swift = (SwiftTO)destDataStore;
+
+ try {
+
+ String containerName = SwiftUtil.getContainerName(destData.getObjectType().toString(), destData.getId());
+ String swiftPath = SwiftUtil.putObject(swift, srcFile, containerName, srcFile.getName());
+
+
+ DataTO retObj = null;
+ if (destData.getObjectType() == DataObjectType.TEMPLATE) {
+ swiftUploadMetadataFile(swift, srcFile, containerName);
+ TemplateObjectTO newTemplate = new TemplateObjectTO();
+ newTemplate.setPath(swiftPath);
+ newTemplate.setSize(getVirtualSize(srcFile, getTemplateFormat(srcFile.getName())));
+ newTemplate.setPhysicalSize(srcFile.length());
+ newTemplate.setFormat(getTemplateFormat(srcFile.getName()));
+ retObj = newTemplate;
+ } else if (destData.getObjectType() == DataObjectType.VOLUME) {
+ VolumeObjectTO newVol = new VolumeObjectTO();
+ newVol.setPath(containerName);
+ newVol.setSize(getVirtualSize(srcFile, getTemplateFormat(srcFile.getName())));
+ retObj = newVol;
+ } else if (destData.getObjectType() == DataObjectType.SNAPSHOT) {
+ SnapshotObjectTO newSnapshot = new SnapshotObjectTO();
+ newSnapshot.setPath(containerName);
+ retObj = newSnapshot;
+ }
+
+ return new CopyCmdAnswer(retObj);
+
+ } catch (Exception e) {
+ s_logger.error("failed to upload " + srcData.getPath(), e);
+ return new CopyCmdAnswer("failed to upload " + srcData.getPath() + e.toString());
+ }
+ }
+
String swiftDownload(SwiftTO swift, String container, String rfilename, String lFullPath) {
Script command = new Script("/bin/bash", s_logger);
command.add("-c");
@@ -1458,13 +1553,13 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S
Map<String, TemplateProp> tmpltInfos = new HashMap<String, TemplateProp>();
for (String container : containers) {
if (container.startsWith("T-")) {
- String[] files = SwiftUtil.list(swift, container, "template.properties");
+ String[] files = SwiftUtil.list(swift, container, _tmpltpp);
if (files.length != 1) {
continue;
}
try {
File tempFile = File.createTempFile("template", ".tmp");
- File tmpFile = SwiftUtil.getObject(swift, tempFile, container + File.separator + "template.properties");
+ File tmpFile = SwiftUtil.getObject(swift, tempFile, container + File.separator + _tmpltpp);
if (tmpFile == null) {
continue;
}
@@ -1779,7 +1874,7 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S
} else {
boolean found = false;
for (File f : tmpltFiles) {
- if (!found && f.getName().equals("template.properties")) {
+ if (!found && f.getName().equals(_tmpltpp)) {
found = true;
}
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/f5ac8ddd/services/secondary-storage/server/test/org/apache/cloudstack/storage/resource/LocalNfsSecondaryStorageResourceTest.java
----------------------------------------------------------------------
diff --git a/services/secondary-storage/server/test/org/apache/cloudstack/storage/resource/LocalNfsSecondaryStorageResourceTest.java b/services/secondary-storage/server/test/org/apache/cloudstack/storage/resource/LocalNfsSecondaryStorageResourceTest.java
index 52bde6a..b33ce3b 100644
--- a/services/secondary-storage/server/test/org/apache/cloudstack/storage/resource/LocalNfsSecondaryStorageResourceTest.java
+++ b/services/secondary-storage/server/test/org/apache/cloudstack/storage/resource/LocalNfsSecondaryStorageResourceTest.java
@@ -18,29 +18,6 @@
*/
package org.apache.cloudstack.storage.resource;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.util.Map;
-import java.util.Properties;
-import java.util.UUID;
-
-import javax.naming.ConfigurationException;
-
-import junit.framework.Assert;
-import junit.framework.TestCase;
-
-import org.apache.log4j.Logger;
-import org.junit.Before;
-import org.junit.Test;
-import org.mockito.Mockito;
-
-import org.apache.cloudstack.storage.command.CopyCmdAnswer;
-import org.apache.cloudstack.storage.command.CopyCommand;
-import org.apache.cloudstack.storage.command.DownloadCommand;
-import org.apache.cloudstack.storage.to.TemplateObjectTO;
-
import com.cloud.agent.api.storage.DownloadAnswer;
import com.cloud.agent.api.storage.ListTemplateAnswer;
import com.cloud.agent.api.storage.ListTemplateCommand;
@@ -51,7 +28,28 @@ import com.cloud.storage.DataStoreRole;
import com.cloud.storage.Storage;
import com.cloud.utils.PropertiesUtil;
import com.cloud.utils.exception.CloudRuntimeException;
+import junit.framework.Assert;
+import junit.framework.TestCase;
+import org.apache.cloudstack.storage.command.CopyCmdAnswer;
+import org.apache.cloudstack.storage.command.CopyCommand;
+import org.apache.cloudstack.storage.command.DownloadCommand;
+import org.apache.cloudstack.storage.to.TemplateObjectTO;
+import org.apache.log4j.Logger;
+import org.junit.Before;
+import org.junit.Ignore;
+import org.junit.Test;
+import org.mockito.Mockito;
+
+import javax.naming.ConfigurationException;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.util.Map;
+import java.util.Properties;
+import java.util.UUID;
+@Ignore
public class LocalNfsSecondaryStorageResourceTest extends TestCase {
private static Map<String, Object> testParams;
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/f5ac8ddd/services/secondary-storage/server/test/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResourceTest.java
----------------------------------------------------------------------
diff --git a/services/secondary-storage/server/test/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResourceTest.java b/services/secondary-storage/server/test/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResourceTest.java
index 13ddb35..e437a0f 100644
--- a/services/secondary-storage/server/test/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResourceTest.java
+++ b/services/secondary-storage/server/test/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResourceTest.java
@@ -18,91 +18,67 @@
*/
package org.apache.cloudstack.storage.resource;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.net.URI;
-import java.util.Map;
-import java.util.Properties;
-
-import javax.naming.ConfigurationException;
-
-import junit.framework.Assert;
-import junit.framework.TestCase;
-
+import com.cloud.test.TestAppender;
+import org.apache.cloudstack.storage.command.DeleteCommand;
+import org.apache.cloudstack.storage.to.TemplateObjectTO;
import org.apache.log4j.Level;
-import org.apache.log4j.Logger;
+import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.Mockito;
+import org.powermock.api.mockito.PowerMockito;
+import org.powermock.core.classloader.annotations.PrepareForTest;
+import org.powermock.modules.junit4.PowerMockRunner;
-import com.cloud.utils.PropertiesUtil;
-import com.cloud.utils.exception.CloudRuntimeException;
+import java.io.BufferedWriter;
+import java.io.FileWriter;
+import java.io.StringWriter;
-public class NfsSecondaryStorageResourceTest extends TestCase {
- private static Map<String, Object> testParams;
+import static org.mockito.Matchers.any;
+import static org.mockito.Mockito.doThrow;
+import static org.mockito.Mockito.spy;
- private static final Logger s_logger = Logger.getLogger(NfsSecondaryStorageResourceTest.class.getName());
+@RunWith(PowerMockRunner.class)
+public class NfsSecondaryStorageResourceTest {
- NfsSecondaryStorageResource resource;
+ private NfsSecondaryStorageResource resource;
@Before
- @Override
- public void setUp() throws ConfigurationException {
- s_logger.setLevel(Level.ALL);
+ public void setUp() {
resource = new NfsSecondaryStorageResource();
- resource.setInSystemVM(true);
- testParams = PropertiesUtil.toMap(loadProperties());
- resource.configureStorageLayerClass(testParams);
- Object testLocalRoot = testParams.get("testLocalRoot");
- if (testLocalRoot != null) {
- resource.setParentPath((String)testLocalRoot);
- }
}
@Test
- public void testMount() throws Exception {
- String sampleUriStr = "cifs://192.168.1.128/CSHV3?user=administrator&password=1pass%40word1&foo=bar";
- URI sampleUri = new URI(sampleUriStr);
-
- s_logger.info("Check HostIp parsing");
- String hostIpStr = resource.getUriHostIp(sampleUri);
- Assert.assertEquals("Expected host IP " + sampleUri.getHost() + " and actual host IP " + hostIpStr + " differ.", sampleUri.getHost(), hostIpStr);
-
- s_logger.info("Check option parsing");
- String expected = "user=administrator,password=1pass@word1,foo=bar,";
- String actualOpts = resource.parseCifsMountOptions(sampleUri);
- Assert.assertEquals("Options should be " + expected + " and not " + actualOpts, expected, actualOpts);
-
- // attempt a configured mount
- final Map<String, Object> params = PropertiesUtil.toMap(loadProperties());
- String sampleMount = (String)params.get("testCifsMount");
- if (!sampleMount.isEmpty()) {
- s_logger.info("functional test, mount " + sampleMount);
- URI realMntUri = new URI(sampleMount);
- String mntSubDir = resource.mountUri(realMntUri);
- s_logger.info("functional test, umount " + mntSubDir);
- resource.umount(resource.getMountingRoot() + mntSubDir, realMntUri);
- } else {
- s_logger.info("no entry for testCifsMount in " + "./conf/agent.properties - skip functional test");
- }
- }
+ @PrepareForTest(NfsSecondaryStorageResource.class)
+ public void testSwiftWriteMetadataFile() throws Exception {
+ String expected = "uniquename=test\nfilename=testfile\nsize=100\nvirtualsize=1000";
- public static Properties loadProperties() throws ConfigurationException {
- Properties properties = new Properties();
- final File file = PropertiesUtil.findConfigFile("agent.properties");
- if (file == null) {
- throw new ConfigurationException("Unable to find agent.properties.");
- }
- s_logger.info("agent.properties found at " + file.getAbsolutePath());
- try(FileInputStream fs = new FileInputStream(file);) {
- properties.load(fs);
- } catch (final FileNotFoundException ex) {
- throw new CloudRuntimeException("Cannot find the file: " + file.getAbsolutePath(), ex);
- } catch (final IOException ex) {
- throw new CloudRuntimeException("IOException in reading " + file.getAbsolutePath(), ex);
- }
- return properties;
+ StringWriter stringWriter = new StringWriter();
+ BufferedWriter bufferWriter = new BufferedWriter(stringWriter);
+ PowerMockito.whenNew(BufferedWriter.class).withArguments(any(FileWriter.class)).thenReturn(bufferWriter);
+
+ resource.swiftWriteMetadataFile("testfile", "test", "testfile", 100, 1000);
+
+ Assert.assertEquals(expected, stringWriter.toString());
}
+ @Test
+ public void testCleanupStagingNfs() throws Exception{
+
+ NfsSecondaryStorageResource spyResource = spy(resource);
+ RuntimeException exception = new RuntimeException();
+ doThrow(exception).when(spyResource).execute(any(DeleteCommand.class));
+ TemplateObjectTO mockTemplate = Mockito.mock(TemplateObjectTO.class);
+
+ TestAppender.TestAppenderBuilder appenderBuilder = new TestAppender.TestAppenderBuilder();
+ appenderBuilder.addExpectedPattern(Level.DEBUG, "Failed to clean up staging area:");
+ TestAppender testLogAppender = appenderBuilder.build();
+ TestAppender.safeAddAppender(NfsSecondaryStorageResource.s_logger, testLogAppender);
+
+ spyResource.cleanupStagingNfs(mockTemplate);
+
+ testLogAppender.assertMessagesLogged();
+
+ }
}