[GitHub] GabrielBrascher commented on a change in pull request #2414: [CLOUDSTACK-10241] Duplicated file SRs being created in XenServer pools

[GitBox <gi...@apache.org>]

GabrielBrascher commented on a change in pull request #2414: [CLOUDSTACK-10241] Duplicated file SRs being created in XenServer pools
URL: https://github.com/apache/cloudstack/pull/2414#discussion_r175882326
 
 

 ##########
 File path: plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/resource/Xenserver625StorageProcessor.java
 ##########
 @@ -65,90 +69,181 @@ public Xenserver625StorageProcessor(final CitrixResourceBase resource) {
         super(resource);
     }
 
-    protected boolean mountNfs(final Connection conn, final String remoteDir, String localDir) {
+    private void mountNfs(Connection conn, String remoteDir, String localDir) {
         if (localDir == null) {
             localDir = "/var/cloud_mount/" + UUID.nameUUIDFromBytes(remoteDir.getBytes());
         }
-
-        final String results = hypervisorResource.callHostPluginAsync(conn, "cloud-plugin-storage", "mountNfsSecondaryStorage", 100 * 1000, "localDir", localDir, "remoteDir",
-                remoteDir);
-
-        if (results == null || results.isEmpty()) {
+        String result = hypervisorResource.callHostPluginAsync(conn, "cloud-plugin-storage", "mountNfsSecondaryStorage", 100 * 1000, "localDir", localDir, "remoteDir", remoteDir);
+        if (StringUtils.isBlank(result)) {
             final String errMsg = "Could not mount secondary storage " + remoteDir + " on host " + localDir;
-
             s_logger.warn(errMsg);
-
             throw new CloudRuntimeException(errMsg);
         }
-
-        return true;
     }
 
-    protected boolean makeDirectory(final Connection conn, final String path) {
-        final String result = hypervisorResource.callHostPlugin(conn, "cloud-plugin-storage", "makeDirectory", "path", path);
+    protected boolean makeDirectory(Connection conn, String path) {
+        String result = hypervisorResource.callHostPlugin(conn, "cloud-plugin-storage", "makeDirectory", "path", path);
+        return StringUtils.isNotBlank(result);
+    }
 
-        if (result == null || result.isEmpty()) {
-            return false;
+    /**
+     *  Creates the file SR for the given path. If there already exist a file SR for the path, we return the existing one.
+     *  This method uses a synchronized block to guarantee that only a single file SR is created per path.
+     *  If it is not possible to retrieve one file SR or to create one, a runtime exception will be thrown.
+     */
+    protected SR createFileSR(Connection conn, String path) {
+        String srPath = StringUtils.trim(path);
+        synchronized (srPath) {
+            SR sr = retrieveAlreadyConfiguredSrWithoutException(conn, srPath);
+            if (sr == null) {
+                sr = createNewFileSr(conn, srPath);
+            }
+            if (sr == null) {
+                String hostUuid = this.hypervisorResource._host.getUuid();
+                throw new CloudRuntimeException(String.format("Could not retrieve an already used file SR for path [%s] or create a new file SR on host [%s]", srPath, hostUuid));
+            }
+            return sr;
         }
-
-        return true;
     }
 
-    protected SR createFileSR(final Connection conn, final String path) {
+    /**
+     * Creates a new file SR for the given path. If any of XenServer's checked exception occurs, we use method {@link #removeSrAndPbdIfPossible(Connection, SR, PBD)} to clean the created PBD and SR entries.
+     * To avoid race conditions between management servers, we are using a deterministic srUuid for the file SR to be created (we are leaving XenServer with the burden of managing race conditions). The UUID is based on the SR file path, and is generated using {@link UUID#nameUUIDFromBytes(byte[])}.
+     * If there is an SR with the generated UUID, this means that some other management server has just created it. An exception will occur and this exception will be an {@link InternalError}. The exception will contain {@link InternalError#message} a message saying 'Db_exn.Uniqueness_constraint_violation'.
+     * For cases where the previous described error happens, we catch the exception and use the method {@link #retrieveAlreadyConfiguredSrWithoutException(Connection, String)}.
+     */
+    protected SR createNewFileSr(Connection conn, String srPath) {
+        String hostUuid = hypervisorResource.getHost().getUuid();
+        s_logger.debug(String.format("Creating file SR for path [%s] on host [%s]", srPath, this.hypervisorResource._host.getUuid()));
         SR sr = null;
         PBD pbd = null;
-
         try {
-            final String srname = path.trim();
-            synchronized (srname.intern()) {
-                final Set<SR> srs = SR.getByNameLabel(conn, srname);
-                if (srs != null && !srs.isEmpty()) {
-                    return srs.iterator().next();
-                }
-                final Map<String, String> smConfig = new HashMap<String, String>();
-                final Host host = Host.getByUuid(conn, hypervisorResource.getHost().getUuid());
-                final String uuid = UUID.randomUUID().toString();
-                sr = SR.introduce(conn, uuid, srname, srname, "file", "file", false, smConfig);
-                final PBD.Record record = new PBD.Record();
-                record.host = host;
-                record.SR = sr;
-                smConfig.put("location", path);
-                record.deviceConfig = smConfig;
-                pbd = PBD.create(conn, record);
-                pbd.plug(conn);
-                sr.scan(conn);
-            }
+            Host host = Host.getByUuid(conn, hostUuid);
+            String srUuid = UUID.nameUUIDFromBytes(srPath.getBytes()).toString();
+
+            Map<String, String> smConfig = new HashMap<String, String>();
+            sr = SR.introduce(conn, srUuid, srPath, srPath, "file", "file", false, smConfig);
+
+            PBD.Record record = new PBD.Record();
+            record.host = host;
+            record.SR = sr;
+            smConfig.put("location", srPath);
+            record.deviceConfig = smConfig;
+            pbd = PBD.create(conn, record);
+            pbd.plug(conn);
+            sr.scan(conn);
             return sr;
-        } catch (final Exception ex) {
-            try {
-                if (pbd != null) {
-                    pbd.destroy(conn);
+        } catch (XenAPIException | XmlRpcException e) {
+            if (e instanceof Types.InternalError) {
+                String expectedDuplicatedFileSrErrorMessage = "Db_exn.Uniqueness_constraint_violation";
+
+                Types.InternalError internalErrorException = (Types.InternalError)e;
+                if (StringUtils.contains(internalErrorException.message, expectedDuplicatedFileSrErrorMessage)) {
+                    s_logger.debug(String.format(
+                            "It seems that we have hit a race condition case here while creating file SR for [%s]. Instead of creating one, we will reuse the one that already exist in the XenServer pool.",
+                            srPath));
+                    return retrieveAlreadyConfiguredSrWithoutException(conn, srPath);
                 }
-            } catch (final Exception e1) {
-                s_logger.debug("Failed to destroy PBD", ex);
             }
+            removeSrAndPbdIfPossible(conn, sr, pbd);
+            s_logger.debug(String.format("Could not create file SR [%s] on host [%s].", srPath, hostUuid), e);
+            return null;
+        }
+    }
 
-            try {
-                if (sr != null) {
-                    sr.forget(conn);
-                }
-            } catch (final Exception e2) {
-                s_logger.error("Failed to forget SR", ex);
-            }
+    /**
+     * Calls {@link #unplugPbd(Connection, PBD)} and {@link #forgetSr(Connection, SR)}, if respective objects are not null.
+     */
+    protected void removeSrAndPbdIfPossible(Connection conn, SR sr, PBD pbd) {
+        if (pbd != null) {
+            unplugPbd(conn, pbd);
+        }
+        if (sr != null) {
+            forgetSr(conn, sr);
+        }
+    }
+
+    /**
+     * This is a simple facade for {@link #retrieveAlreadyConfiguredSr(Connection, String)} method.
+     * If we catch any of the checked exception of {@link #retrieveAlreadyConfiguredSr(Connection, String)}, we re-throw as a {@link CloudRuntimeException}.
+     */
+    protected SR retrieveAlreadyConfiguredSrWithoutException(Connection conn, String srPath) {
+        try {
+            return retrieveAlreadyConfiguredSr(conn, srPath);
+        } catch (XenAPIException | XmlRpcException e) {
+            throw new CloudRuntimeException("Unexpected exception while trying to retrieve an already configured file SR for path: " + srPath);
+        }
+    }
 
-            final String msg = "createFileSR failed! due to the following: " + ex.toString();
+    /**
+     *  This method will check if there is an already configured file SR for the given path. If by any chance we find more an one SR with the same name (mount point path) we throw a runtime exception because this situation should never happen.
 
 Review comment:
   If by any chance we find more **an** one SR with the same name -> more **than** one

----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
users@infra.apache.org


With regards,
Apache Git Services