You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@stratos.apache.org by ra...@apache.org on 2015/11/24 11:12:16 UTC

stratos git commit: Initialize locks on complete applications event in metadata service

Repository: stratos
Updated Branches:
  refs/heads/stratos-4.1.x edb172b68 -> e4a2af690


Initialize locks on complete applications event in metadata service


Project: http://git-wip-us.apache.org/repos/asf/stratos/repo
Commit: http://git-wip-us.apache.org/repos/asf/stratos/commit/e4a2af69
Tree: http://git-wip-us.apache.org/repos/asf/stratos/tree/e4a2af69
Diff: http://git-wip-us.apache.org/repos/asf/stratos/diff/e4a2af69

Branch: refs/heads/stratos-4.1.x
Commit: e4a2af690ef6f354203c97ca1d0d920ad63cdb16
Parents: edb172b
Author: Akila Perera <ra...@gmail.com>
Authored: Tue Nov 24 15:41:30 2015 +0530
Committer: Akila Perera <ra...@gmail.com>
Committed: Tue Nov 24 15:42:00 2015 +0530

----------------------------------------------------------------------
 .../stratos/metadata/service/Constants.java     |   3 +-
 .../MetadataApplicationEventReceiver.java       |  93 ++++++++++
 .../service/MetadataTopologyEventReceiver.java  |  12 +-
 .../service/exception/MetadataException.java    |   4 +
 .../metadata/service/registry/DataStore.java    |   4 +-
 .../service/registry/MetadataApiRegistry.java   | 180 ++++++++++++-------
 .../integration/common/RestConstants.java       |   2 +
 .../integration/common/TopologyHandler.java     |   2 +
 .../integration/common/rest/RestClient.java     |  21 ++-
 9 files changed, 236 insertions(+), 85 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/stratos/blob/e4a2af69/components/org.apache.stratos.metadata.service/src/main/java/org/apache/stratos/metadata/service/Constants.java
----------------------------------------------------------------------
diff --git a/components/org.apache.stratos.metadata.service/src/main/java/org/apache/stratos/metadata/service/Constants.java b/components/org.apache.stratos.metadata.service/src/main/java/org/apache/stratos/metadata/service/Constants.java
index 0027a47..4fe222e 100644
--- a/components/org.apache.stratos.metadata.service/src/main/java/org/apache/stratos/metadata/service/Constants.java
+++ b/components/org.apache.stratos.metadata.service/src/main/java/org/apache/stratos/metadata/service/Constants.java
@@ -22,7 +22,6 @@ package org.apache.stratos.metadata.service;
  * Global constants used in this module
  */
 public class Constants {
-
     public static final String METADATASERVICE_CONFIG_FILE_NAME = "metadataservice.xml";
-
+    public static final String METADATA_SERVICE_THREAD_POOL_ID = "metadata.service.thread.pool.";
 }

http://git-wip-us.apache.org/repos/asf/stratos/blob/e4a2af69/components/org.apache.stratos.metadata.service/src/main/java/org/apache/stratos/metadata/service/MetadataApplicationEventReceiver.java
----------------------------------------------------------------------
diff --git a/components/org.apache.stratos.metadata.service/src/main/java/org/apache/stratos/metadata/service/MetadataApplicationEventReceiver.java b/components/org.apache.stratos.metadata.service/src/main/java/org/apache/stratos/metadata/service/MetadataApplicationEventReceiver.java
new file mode 100644
index 0000000..ca00c3c
--- /dev/null
+++ b/components/org.apache.stratos.metadata.service/src/main/java/org/apache/stratos/metadata/service/MetadataApplicationEventReceiver.java
@@ -0,0 +1,93 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.stratos.metadata.service;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.stratos.common.concurrent.locks.ReadWriteLock;
+import org.apache.stratos.common.threading.StratosThreadPool;
+import org.apache.stratos.messaging.domain.application.Application;
+import org.apache.stratos.messaging.domain.application.ApplicationStatus;
+import org.apache.stratos.messaging.domain.application.Applications;
+import org.apache.stratos.messaging.event.Event;
+import org.apache.stratos.messaging.event.application.CompleteApplicationsEvent;
+import org.apache.stratos.messaging.listener.application.CompleteApplicationsEventListener;
+import org.apache.stratos.messaging.message.receiver.application.ApplicationsEventReceiver;
+import org.apache.stratos.metadata.service.registry.MetadataApiRegistry;
+
+import java.util.Map;
+import java.util.concurrent.ExecutorService;
+
+/**
+ * Application receiver class for metadata service
+ */
+public class MetadataApplicationEventReceiver {
+    private static final Log log = LogFactory.getLog(MetadataApplicationEventReceiver.class);
+    private ApplicationsEventReceiver applicationsEventReceiver;
+    private ExecutorService executorService;
+
+    public MetadataApplicationEventReceiver() {
+        this.applicationsEventReceiver = new ApplicationsEventReceiver();
+        executorService = StratosThreadPool.getExecutorService(Constants.METADATA_SERVICE_THREAD_POOL_ID, 20);
+        addEventListeners();
+    }
+
+    private void addEventListeners() {
+        applicationsEventReceiver.addEventListener(new CompleteApplicationsEventListener() {
+            @Override
+            protected void onEvent(Event event) {
+                CompleteApplicationsEvent completeApplicationsEvent = (CompleteApplicationsEvent) event;
+                Applications applications = completeApplicationsEvent.getApplications();
+                for (Map.Entry<String, Application> entry : applications.getApplications().entrySet()) {
+                    ApplicationStatus status = entry.getValue().getStatus();
+                    String appId = entry.getKey();
+                    if (ApplicationStatus.Active.equals(status) || ApplicationStatus.Inactive.equals(status)) {
+                        MetadataApiRegistry.getApplicationIdToReadWriteLockMap().putIfAbsent(appId,
+                                new ReadWriteLock(Constants.METADATA_SERVICE_THREAD_POOL_ID.concat(appId)));
+                    }
+                }
+                if (log.isDebugEnabled()) {
+                    log.debug("Metadata service READ WRITE locks initialized on complete applications event.");
+                }
+                terminate();
+            }
+        });
+    }
+
+    public void execute() {
+        applicationsEventReceiver.setExecutorService(getExecutorService());
+        applicationsEventReceiver.execute();
+
+        if (log.isInfoEnabled()) {
+            log.info("Metadata service application receiver started.");
+        }
+    }
+
+    public void terminate() {
+        applicationsEventReceiver.terminate();
+        if (log.isInfoEnabled()) {
+            log.info("Metadata service application receiver stopped.");
+        }
+    }
+
+    public ExecutorService getExecutorService() {
+        return executorService;
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/stratos/blob/e4a2af69/components/org.apache.stratos.metadata.service/src/main/java/org/apache/stratos/metadata/service/MetadataTopologyEventReceiver.java
----------------------------------------------------------------------
diff --git a/components/org.apache.stratos.metadata.service/src/main/java/org/apache/stratos/metadata/service/MetadataTopologyEventReceiver.java b/components/org.apache.stratos.metadata.service/src/main/java/org/apache/stratos/metadata/service/MetadataTopologyEventReceiver.java
index f483995..e516271 100644
--- a/components/org.apache.stratos.metadata.service/src/main/java/org/apache/stratos/metadata/service/MetadataTopologyEventReceiver.java
+++ b/components/org.apache.stratos.metadata.service/src/main/java/org/apache/stratos/metadata/service/MetadataTopologyEventReceiver.java
@@ -23,12 +23,8 @@ import org.apache.commons.logging.LogFactory;
 import org.apache.stratos.common.concurrent.locks.ReadWriteLock;
 import org.apache.stratos.common.threading.StratosThreadPool;
 import org.apache.stratos.messaging.event.Event;
-import org.apache.stratos.messaging.event.application.ApplicationCreatedEvent;
-import org.apache.stratos.messaging.event.application.ApplicationDeletedEvent;
 import org.apache.stratos.messaging.event.topology.ApplicationClustersCreatedEvent;
 import org.apache.stratos.messaging.event.topology.ApplicationClustersRemovedEvent;
-import org.apache.stratos.messaging.listener.application.ApplicationCreatedEventListener;
-import org.apache.stratos.messaging.listener.application.ApplicationDeletedEventListener;
 import org.apache.stratos.messaging.listener.topology.ApplicationClustersCreatedEventListener;
 import org.apache.stratos.messaging.listener.topology.ApplicationClustersRemovedEventListener;
 import org.apache.stratos.messaging.message.receiver.topology.TopologyEventReceiver;
@@ -41,14 +37,12 @@ import java.util.concurrent.ExecutorService;
  */
 public class MetadataTopologyEventReceiver {
     private static final Log log = LogFactory.getLog(MetadataTopologyEventReceiver.class);
-
     private TopologyEventReceiver topologyEventReceiver;
     private ExecutorService executorService;
-    public static final String METADATA_SERVICE_THREAD_POOL_ID = "metadata.service.thread.pool.";
 
     public MetadataTopologyEventReceiver() {
         this.topologyEventReceiver = new TopologyEventReceiver();
-        executorService = StratosThreadPool.getExecutorService(METADATA_SERVICE_THREAD_POOL_ID, 10);
+        executorService = StratosThreadPool.getExecutorService(Constants.METADATA_SERVICE_THREAD_POOL_ID, 20);
         addEventListeners();
     }
 
@@ -58,8 +52,8 @@ public class MetadataTopologyEventReceiver {
             protected void onEvent(Event event) {
                 ApplicationClustersCreatedEvent appClustersCreatedEvent = (ApplicationClustersCreatedEvent) event;
                 String applicationId = appClustersCreatedEvent.getAppId();
-                MetadataApiRegistry.getApplicationIdToReadWriteLockMap()
-                        .put(applicationId, new ReadWriteLock(METADATA_SERVICE_THREAD_POOL_ID.concat(applicationId)));
+                MetadataApiRegistry.getApplicationIdToReadWriteLockMap().put(applicationId,
+                        new ReadWriteLock(Constants.METADATA_SERVICE_THREAD_POOL_ID.concat(applicationId)));
             }
         });
 

http://git-wip-us.apache.org/repos/asf/stratos/blob/e4a2af69/components/org.apache.stratos.metadata.service/src/main/java/org/apache/stratos/metadata/service/exception/MetadataException.java
----------------------------------------------------------------------
diff --git a/components/org.apache.stratos.metadata.service/src/main/java/org/apache/stratos/metadata/service/exception/MetadataException.java b/components/org.apache.stratos.metadata.service/src/main/java/org/apache/stratos/metadata/service/exception/MetadataException.java
index 8d6fafa..38a5b36 100644
--- a/components/org.apache.stratos.metadata.service/src/main/java/org/apache/stratos/metadata/service/exception/MetadataException.java
+++ b/components/org.apache.stratos.metadata.service/src/main/java/org/apache/stratos/metadata/service/exception/MetadataException.java
@@ -24,4 +24,8 @@ public class MetadataException extends Exception {
     public MetadataException(String message, Throwable cause) {
         super(message, cause);
     }
+
+    public MetadataException(String message) {
+        super(message);
+    }
 }

http://git-wip-us.apache.org/repos/asf/stratos/blob/e4a2af69/components/org.apache.stratos.metadata.service/src/main/java/org/apache/stratos/metadata/service/registry/DataStore.java
----------------------------------------------------------------------
diff --git a/components/org.apache.stratos.metadata.service/src/main/java/org/apache/stratos/metadata/service/registry/DataStore.java b/components/org.apache.stratos.metadata.service/src/main/java/org/apache/stratos/metadata/service/registry/DataStore.java
index a6415bd..a71c745 100644
--- a/components/org.apache.stratos.metadata.service/src/main/java/org/apache/stratos/metadata/service/registry/DataStore.java
+++ b/components/org.apache.stratos.metadata.service/src/main/java/org/apache/stratos/metadata/service/registry/DataStore.java
@@ -31,9 +31,9 @@ public interface DataStore {
 
     public void addPropertyToApplication(String applicationId, Property property) throws RegistryException;
 
-    public List<Property> getApplicationProperties(String applicationName) throws RegistryException;
+    public List<Property> getApplicationProperties(String applicationId) throws RegistryException;
 
-    public List<Property> getClusterProperties(String applicationName, String clusterId)
+    public List<Property> getClusterProperties(String applicationId, String clusterId)
             throws RegistryException;
 
     public void addPropertyToCluster(String applicationId, String clusterId, Property property) throws RegistryException;

http://git-wip-us.apache.org/repos/asf/stratos/blob/e4a2af69/components/org.apache.stratos.metadata.service/src/main/java/org/apache/stratos/metadata/service/registry/MetadataApiRegistry.java
----------------------------------------------------------------------
diff --git a/components/org.apache.stratos.metadata.service/src/main/java/org/apache/stratos/metadata/service/registry/MetadataApiRegistry.java b/components/org.apache.stratos.metadata.service/src/main/java/org/apache/stratos/metadata/service/registry/MetadataApiRegistry.java
index 7531b16..11ab2d2 100644
--- a/components/org.apache.stratos.metadata.service/src/main/java/org/apache/stratos/metadata/service/registry/MetadataApiRegistry.java
+++ b/components/org.apache.stratos.metadata.service/src/main/java/org/apache/stratos/metadata/service/registry/MetadataApiRegistry.java
@@ -22,9 +22,11 @@ import org.apache.commons.lang.StringUtils;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.stratos.common.concurrent.locks.ReadWriteLock;
+import org.apache.stratos.metadata.service.MetadataApplicationEventReceiver;
 import org.apache.stratos.metadata.service.MetadataTopologyEventReceiver;
 import org.apache.stratos.metadata.service.ServiceHolder;
 import org.apache.stratos.metadata.service.definition.Property;
+import org.apache.stratos.metadata.service.exception.MetadataException;
 import org.wso2.carbon.context.PrivilegedCarbonContext;
 import org.wso2.carbon.registry.core.Registry;
 import org.wso2.carbon.registry.core.Resource;
@@ -35,6 +37,7 @@ import org.wso2.carbon.utils.multitenancy.MultitenantConstants;
 import javax.servlet.http.HttpServletRequest;
 import javax.ws.rs.core.Context;
 import java.util.*;
+import java.util.concurrent.ConcurrentHashMap;
 
 /**
  * Carbon registry implementation
@@ -46,57 +49,75 @@ public class MetadataApiRegistry implements DataStore {
     private static Log log = LogFactory.getLog(MetadataApiRegistry.class);
     @Context
     HttpServletRequest httpServletRequest;
-    private static final Map<String, ReadWriteLock> applicationIdToReadWriteLockMap = new HashMap<>();
+    private static final ConcurrentHashMap<String, ReadWriteLock> applicationIdToReadWriteLockMap
+            = new ConcurrentHashMap<>();
     private MetadataTopologyEventReceiver metadataTopologyEventReceiver;
+    private MetadataApplicationEventReceiver metadataApplicationEventReceiver;
 
     public MetadataApiRegistry() {
         metadataTopologyEventReceiver = new MetadataTopologyEventReceiver();
         metadataTopologyEventReceiver.execute();
-    }
-
-    public List<Property> getApplicationProperties(String applicationName) throws RegistryException {
-        Registry tempRegistry = getRegistry();
-        String resourcePath = mainResource + applicationName;
 
-        if (!tempRegistry.resourceExists(resourcePath)) {
-            return null;
-        }
-        // We are using only super tenant registry to persist
-        PrivilegedCarbonContext ctx = PrivilegedCarbonContext.getThreadLocalCarbonContext();
-        ctx.setTenantId(MultitenantConstants.SUPER_TENANT_ID);
-        ctx.setTenantDomain(MultitenantConstants.SUPER_TENANT_DOMAIN_NAME);
-
-        Resource regResource = tempRegistry.get(resourcePath);
-        ArrayList<Property> newProperties = new ArrayList<Property>();
-
-        Properties props = regResource.getProperties();
-        Enumeration<?> x = props.propertyNames();
-        while (x.hasMoreElements()) {
-            String key = (String) x.nextElement();
-            List<String> values = regResource.getPropertyValues(key);
-            Property property = new Property();
-            property.setKey(key);
-            String[] valueArr = new String[values.size()];
-            property.setValues(values.toArray(valueArr));
+        metadataApplicationEventReceiver = new MetadataApplicationEventReceiver();
+        metadataApplicationEventReceiver.execute();
+    }
 
-            newProperties.add(property);
+    /**
+     * Get Properties of application
+     *
+     * @param applicationId Application ID under which properties should be retrieved
+     * @return List of properties
+     * @throws RegistryException
+     */
+    public List<Property> getApplicationProperties(String applicationId) throws RegistryException {
+        String resourcePath = mainResource + applicationId;
+        try {
+            acquireReadLock(applicationId);
+            return getRegistryResourceProperties(resourcePath, applicationId);
+        } catch (Exception e) {
+            String msg = String.format("Failed to get application properties from registry [resource-path] %s for "
+                    + "[application-id] %s", resourcePath, applicationId);
+            log.error(msg, e);
+            throw new RegistryException(msg, e);
+        } finally {
+            try {
+                releaseReadLock(applicationId);
+            } catch (MetadataException ignored) {
+            }
         }
-        return newProperties;
     }
 
     /**
-     * Get Properties of clustor
+     * Get properties of cluster
      *
-     * @param applicationName
-     * @param clusterId
-     * @return
+     * @param applicationId Application ID under which properties should be retrieved
+     * @param clusterId     Cluster ID under which properties should be retrieved
+     * @return List of properties
      * @throws RegistryException
      */
-    public List<Property> getClusterProperties(String applicationName, String clusterId) throws RegistryException {
-        Registry tempRegistry = getRegistry();
-        String resourcePath = mainResource + applicationName + "/" + clusterId;
+    public List<Property> getClusterProperties(String applicationId, String clusterId) throws RegistryException {
+        String resourcePath = mainResource + applicationId + "/" + clusterId;
+        try {
+            acquireReadLock(applicationId);
+            return getRegistryResourceProperties(resourcePath, applicationId);
+        } catch (Exception e) {
+            String msg = String
+                    .format("Failed to get cluster properties from registry [resource-path] %s for [application-id] "
+                            + "%s, [cluster-id] %s", resourcePath, applicationId, clusterId);
+            log.error(msg, e);
+            throw new RegistryException(msg, e);
+        } finally {
+            try {
+                releaseReadLock(applicationId);
+            } catch (MetadataException ignored) {
+            }
+        }
+    }
 
-        if (!tempRegistry.resourceExists(resourcePath)) {
+    private List<Property> getRegistryResourceProperties(String registryResourcePath, String applicationId)
+            throws RegistryException, MetadataException {
+        Registry tempRegistry = getRegistry();
+        if (!tempRegistry.resourceExists(registryResourcePath)) {
             return null;
         }
 
@@ -105,10 +126,8 @@ public class MetadataApiRegistry implements DataStore {
         ctx.setTenantId(MultitenantConstants.SUPER_TENANT_ID);
         ctx.setTenantDomain(MultitenantConstants.SUPER_TENANT_DOMAIN_NAME);
 
-        Resource regResource = tempRegistry.get(resourcePath);
-
-        ArrayList<Property> newProperties = new ArrayList<Property>();
-
+        Resource regResource = tempRegistry.get(registryResourcePath);
+        ArrayList<Property> newProperties = new ArrayList<>();
         Properties props = regResource.getProperties();
         Enumeration<?> x = props.propertyNames();
         while (x.hasMoreElements()) {
@@ -121,20 +140,20 @@ public class MetadataApiRegistry implements DataStore {
 
             newProperties.add(property);
         }
-
         return newProperties;
     }
 
     public void addPropertyToApplication(String applicationId, Property property) throws RegistryException {
         Registry registry = getRegistry();
         String resourcePath = mainResource + applicationId;
-        acquireWriteLock(applicationId);
+
         try {
+            acquireWriteLock(applicationId);
             // We are using only super tenant registry to persist
             PrivilegedCarbonContext ctx = PrivilegedCarbonContext.getThreadLocalCarbonContext();
             ctx.setTenantId(MultitenantConstants.SUPER_TENANT_ID);
             ctx.setTenantDomain(MultitenantConstants.SUPER_TENANT_DOMAIN_NAME);
-            Resource nodeResource = null;
+            Resource nodeResource;
             if (registry.resourceExists(resourcePath)) {
                 nodeResource = registry.get(resourcePath);
             } else {
@@ -150,8 +169,7 @@ public class MetadataApiRegistry implements DataStore {
                     updated = true;
                     if (log.isDebugEnabled()) {
                         log.debug(String.format("Registry property is added: [resource-path] %s "
-                                        + "[Property Name] %s [Property Value] %s", resourcePath, property.getKey(),
-                                value));
+                                + "[Property Name] %s [Property Value] %s", resourcePath, property.getKey(), value));
                     }
                     nodeResource.addProperty(property.getKey(), value);
                 } else {
@@ -174,7 +192,10 @@ public class MetadataApiRegistry implements DataStore {
             log.error(msg, e);
             throw new RegistryException(msg, e);
         } finally {
-            releaseWriteLock(applicationId);
+            try {
+                releaseWriteLock(applicationId);
+            } catch (MetadataException ignored) {
+            }
         }
     }
 
@@ -188,8 +209,9 @@ public class MetadataApiRegistry implements DataStore {
             throws RegistryException {
         Registry registry = getRegistry();
         String resourcePath = mainResource + applicationId;
-        acquireWriteLock(applicationId);
+
         try {
+            acquireWriteLock(applicationId);
             // We are using only super tenant registry to persist
             PrivilegedCarbonContext ctx = PrivilegedCarbonContext.getThreadLocalCarbonContext();
             ctx.setTenantId(MultitenantConstants.SUPER_TENANT_ID);
@@ -209,29 +231,33 @@ public class MetadataApiRegistry implements DataStore {
         } catch (Exception e) {
             throw new RegistryException("Could not remove registry resource: [resource-path] " + resourcePath, e);
         } finally {
-            releaseWriteLock(applicationId);
+            try {
+                releaseWriteLock(applicationId);
+            } catch (MetadataException ignored) {
+            }
         }
     }
 
     /**
      * Add property to cluster
      *
-     * @param applicationId
-     * @param clusterId
-     * @param property
+     * @param applicationId Application ID against which added property will be stored
+     * @param clusterId     Cluster ID against which added property will be stored
+     * @param property      Property to be stored in the registry
      * @throws RegistryException
      */
     public void addPropertyToCluster(String applicationId, String clusterId, Property property)
             throws RegistryException {
         Registry registry = getRegistry();
         String resourcePath = mainResource + applicationId + "/" + clusterId;
-        acquireWriteLock(applicationId);
+
         try {
+            acquireWriteLock(applicationId);
             // We are using only super tenant registry to persist
             PrivilegedCarbonContext ctx = PrivilegedCarbonContext.getThreadLocalCarbonContext();
             ctx.setTenantId(MultitenantConstants.SUPER_TENANT_ID);
             ctx.setTenantDomain(MultitenantConstants.SUPER_TENANT_DOMAIN_NAME);
-            Resource nodeResource = null;
+            Resource nodeResource;
             if (registry.resourceExists(resourcePath)) {
                 nodeResource = registry.get(resourcePath);
             } else {
@@ -245,8 +271,14 @@ public class MetadataApiRegistry implements DataStore {
             log.info(String.format(
                     "Registry property is persisted: [resource-path] %s [Property Name] %s [Property Values] %s",
                     resourcePath, property.getKey(), Arrays.asList(property.getValues())));
+        } catch (Exception e) {
+            throw new RegistryException("Could not add registry resource: [resource-path] " + resourcePath, e);
+
         } finally {
-            releaseWriteLock(applicationId);
+            try {
+                releaseWriteLock(applicationId);
+            } catch (MetadataException ignored) {
+            }
         }
     }
 
@@ -267,8 +299,9 @@ public class MetadataApiRegistry implements DataStore {
         }
         Registry registry = getRegistry();
         String resourcePath = mainResource + applicationId;
-        acquireWriteLock(applicationId);
+
         try {
+            acquireWriteLock(applicationId);
             // We are using only super tenant registry to persist
             PrivilegedCarbonContext ctx = PrivilegedCarbonContext.getThreadLocalCarbonContext();
             ctx.setTenantId(MultitenantConstants.SUPER_TENANT_ID);
@@ -282,7 +315,10 @@ public class MetadataApiRegistry implements DataStore {
         } catch (Exception e) {
             throw new RegistryException("Could not remove registry resource: [resource-path] " + resourcePath, e);
         } finally {
-            releaseWriteLock(applicationId);
+            try {
+                releaseWriteLock(applicationId);
+            } catch (MetadataException ignored) {
+            }
         }
     }
 
@@ -290,9 +326,10 @@ public class MetadataApiRegistry implements DataStore {
             throws org.wso2.carbon.registry.api.RegistryException {
         Registry registry = getRegistry();
         String resourcePath = mainResource + applicationId;
-        acquireWriteLock(applicationId);
         Resource nodeResource;
+
         try {
+            acquireWriteLock(applicationId);
             // We are using only super tenant registry to persist
             PrivilegedCarbonContext ctx = PrivilegedCarbonContext.getThreadLocalCarbonContext();
             ctx.setTenantId(MultitenantConstants.SUPER_TENANT_ID);
@@ -315,14 +352,19 @@ public class MetadataApiRegistry implements DataStore {
             log.info(String.format("Application [application-id] %s property [property-name] %s removed from Registry ",
                     applicationId, propertyName));
             return true;
+        } catch (Exception e) {
+            throw new RegistryException("Could not remove registry resource: [resource-path] " + resourcePath, e);
         } finally {
-            releaseWriteLock(applicationId);
+            try {
+                releaseWriteLock(applicationId);
+            } catch (MetadataException ignored) {
+            }
         }
     }
 
-    public void acquireReadLock(String applicationId) {
+    public void acquireReadLock(String applicationId) throws MetadataException {
         if (applicationIdToReadWriteLockMap.get(applicationId) == null) {
-            throw new RuntimeException(
+            throw new MetadataException(
                     String.format("Invalid application [application-id] %s not found. Failed to acquire read lock.",
                             applicationId));
         } else {
@@ -330,9 +372,9 @@ public class MetadataApiRegistry implements DataStore {
         }
     }
 
-    public void acquireWriteLock(String applicationId) {
+    public void acquireWriteLock(String applicationId) throws MetadataException {
         if (applicationIdToReadWriteLockMap.get(applicationId) == null) {
-            throw new RuntimeException(
+            throw new MetadataException(
                     String.format("Invalid application [application-id] %s not found. Failed to acquire write lock.",
                             applicationId));
         } else {
@@ -340,9 +382,9 @@ public class MetadataApiRegistry implements DataStore {
         }
     }
 
-    public void releaseReadLock(String applicationId) {
+    public void releaseReadLock(String applicationId) throws MetadataException {
         if (applicationIdToReadWriteLockMap.get(applicationId) == null) {
-            throw new RuntimeException(
+            throw new MetadataException(
                     String.format("Invalid application [application-id] %s not found. Failed to release read lock.",
                             applicationId));
         } else {
@@ -350,9 +392,9 @@ public class MetadataApiRegistry implements DataStore {
         }
     }
 
-    public void releaseWriteLock(String applicationId) {
+    public void releaseWriteLock(String applicationId) throws MetadataException {
         if (applicationIdToReadWriteLockMap.get(applicationId) == null) {
-            throw new RuntimeException(
+            throw new MetadataException(
                     String.format("Invalid application [application-id] %s not found. Failed to release write lock.",
                             applicationId));
         } else {
@@ -360,11 +402,15 @@ public class MetadataApiRegistry implements DataStore {
         }
     }
 
-    public static Map<String, ReadWriteLock> getApplicationIdToReadWriteLockMap() {
+    public static ConcurrentHashMap<String, ReadWriteLock> getApplicationIdToReadWriteLockMap() {
         return applicationIdToReadWriteLockMap;
     }
 
     public void stopTopologyReceiver() {
         metadataTopologyEventReceiver.terminate();
     }
+
+    public void stopApplicationReceiver() {
+        metadataApplicationEventReceiver.terminate();
+    }
 }

http://git-wip-us.apache.org/repos/asf/stratos/blob/e4a2af69/products/stratos/modules/integration/test-common/src/main/java/org/apache/stratos/integration/common/RestConstants.java
----------------------------------------------------------------------
diff --git a/products/stratos/modules/integration/test-common/src/main/java/org/apache/stratos/integration/common/RestConstants.java b/products/stratos/modules/integration/test-common/src/main/java/org/apache/stratos/integration/common/RestConstants.java
index 76ca673..aa2a220 100644
--- a/products/stratos/modules/integration/test-common/src/main/java/org/apache/stratos/integration/common/RestConstants.java
+++ b/products/stratos/modules/integration/test-common/src/main/java/org/apache/stratos/integration/common/RestConstants.java
@@ -52,4 +52,6 @@ public class RestConstants {
     public static final String APPLICATION_POLICIES_PATH = "/application-policies/";
     public static final String APPLICATION_POLICIES_NAME = "applicationPolicy";
 
+    public static final String METADATA_RESPONSE_ATTRIBUTE_KEY = "key";
+    public static final String METADATA_RESPONSE_ATTRIBUTE_VALUES = "values";
 }

http://git-wip-us.apache.org/repos/asf/stratos/blob/e4a2af69/products/stratos/modules/integration/test-common/src/main/java/org/apache/stratos/integration/common/TopologyHandler.java
----------------------------------------------------------------------
diff --git a/products/stratos/modules/integration/test-common/src/main/java/org/apache/stratos/integration/common/TopologyHandler.java b/products/stratos/modules/integration/test-common/src/main/java/org/apache/stratos/integration/common/TopologyHandler.java
index e4650e4..209ad13 100644
--- a/products/stratos/modules/integration/test-common/src/main/java/org/apache/stratos/integration/common/TopologyHandler.java
+++ b/products/stratos/modules/integration/test-common/src/main/java/org/apache/stratos/integration/common/TopologyHandler.java
@@ -235,6 +235,8 @@ public class TopologyHandler {
         Application application = ApplicationManager.getApplications().getApplication(applicationName);
         while (!((application != null) && (application.getStatus() == status))) {
             try {
+                log.info(String.format("Waiting for [application] %s to become [status] %s...", applicationName,
+                        status));
                 Thread.sleep(1000);
             } catch (InterruptedException ignore) {
             }

http://git-wip-us.apache.org/repos/asf/stratos/blob/e4a2af69/products/stratos/modules/integration/test-common/src/main/java/org/apache/stratos/integration/common/rest/RestClient.java
----------------------------------------------------------------------
diff --git a/products/stratos/modules/integration/test-common/src/main/java/org/apache/stratos/integration/common/rest/RestClient.java b/products/stratos/modules/integration/test-common/src/main/java/org/apache/stratos/integration/common/rest/RestClient.java
index d65924c..c996eb8 100644
--- a/products/stratos/modules/integration/test-common/src/main/java/org/apache/stratos/integration/common/rest/RestClient.java
+++ b/products/stratos/modules/integration/test-common/src/main/java/org/apache/stratos/integration/common/rest/RestClient.java
@@ -496,15 +496,26 @@ public class RestClient {
         public PropertyBean deserialize(JsonElement json, Type typeOfT, JsonDeserializationContext context)
                 throws JsonParseException {
             JsonObject jObj = json.getAsJsonObject();
-            JsonElement jElement = jObj.get("values");
+            JsonElement jsonElement = jObj.get(RestConstants.METADATA_RESPONSE_ATTRIBUTE_VALUES);
+            JsonPrimitive jsonPrimitive = jObj.getAsJsonPrimitive(RestConstants.METADATA_RESPONSE_ATTRIBUTE_KEY);
             List<String> tags = new ArrayList<>();
-            if (jElement.isJsonArray()) {
-                tags = context.deserialize(jElement.getAsJsonArray(), new TypeToken<List<String>>() {
+
+            // Json structure is invalid. Return an empty property
+            if (jsonPrimitive == null) {
+                return new PropertyBean("", "");
+            }
+            // Json structure is invalid. Return a property with empty values list
+            String metadataKey = jsonPrimitive.getAsString();
+            if (jsonElement == null) {
+                return new PropertyBean(metadataKey, tags);
+            }
+            if (jsonElement.isJsonArray()) {
+                tags = context.deserialize(jsonElement.getAsJsonArray(), new TypeToken<List<String>>() {
                 }.getType());
             } else {
-                tags.add(jObj.getAsJsonPrimitive("values").getAsString());
+                tags.add(metadataKey);
             }
-            return new PropertyBean(jObj.getAsJsonPrimitive("key").getAsString(), tags);
+            return new PropertyBean(metadataKey, tags);
         }
     }
 }