You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cloudstack.apache.org by bh...@apache.org on 2017/12/02 18:46:28 UTC
[cloudstack] branch debian9-systemvmtemplate updated (c31d4f2 ->
a055e9a)
This is an automated email from the ASF dual-hosted git repository.
bhaisaab pushed a change to branch debian9-systemvmtemplate
in repository https://gitbox.apache.org/repos/asf/cloudstack.git.
discard c31d4f2 CLOUDSTACK-10013: Make the generated VR/json files unique (ports #1470)
new a055e9a CLOUDSTACK-10013: Make the generated VR/json files unique (ports #1470)
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (c31d4f2)
\
N -- N -- N refs/heads/debian9-systemvmtemplate (a055e9a)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
systemvm/patches/debian/opt/cloud/bin/configure.py | 19 +++++++------------
1 file changed, 7 insertions(+), 12 deletions(-)
--
To stop receiving notification emails like this one, please contact
['"commits@cloudstack.apache.org" <co...@cloudstack.apache.org>'].
[cloudstack] 01/01: CLOUDSTACK-10013: Make the generated VR/json
files unique (ports #1470)
Posted by bh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
bhaisaab pushed a commit to branch debian9-systemvmtemplate
in repository https://gitbox.apache.org/repos/asf/cloudstack.git
commit a055e9ae123a9e9d1aa1ad14a20505deab1de9e6
Author: Rohit Yadav <ro...@shapeblue.com>
AuthorDate: Sat Dec 2 23:19:14 2017 +0530
CLOUDSTACK-10013: Make the generated VR/json files unique (ports #1470)
This ports PR #1470 by @remibergsma.
Make the generated json files unique to prevent concurrency issues:
The json files now have UUIDs to prevent them from getting overwritten
before they've been executed. Prevents config to be pushed to the wrong
router.
2016-02-25 18:32:23,797 DEBUG [c.c.a.t.Request] (AgentManager-Handler-1:null) (logid:) Seq 2-4684025087442026584: Processing: { Ans: , MgmtId: 90520732674657, via: 2, Ver: v1, Flags: 10, [{"com.cloud.agent.api.routing.GroupA
nswer":{"results":["null - success: null","null - success: [INFO] update_config.py :: Processing incoming file => vm_dhcp_entry.json.4ea45061-2efb-4467-8eaa-db3d77fb0a7b\n[INFO] Processing JSON file vm_dhcp_entry.json.4ea4506
1-2efb-4467-8eaa-db3d77fb0a7b\n"],"result":true,"wait":0}}] }
On the router:
2016-02-25 18:32:23,416 merge.py __moveFile:298 Processed file written to /var/cache/cloud/processed/vm_dhcp_entry.json.4ea45061-2efb-4467-8eaa-db3d77fb0a7b.gz
Signed-off-by: Rohit Yadav <ro...@shapeblue.com>
---
.../facade/AbstractConfigItemFacade.java | 21 ++-
.../kvm/resource/LibvirtComputingResource.java | 5 +
systemvm/patches/debian/opt/cloud/bin/configure.py | 210 ++++++++++-----------
systemvm/patches/debian/opt/cloud/bin/merge.py | 37 ++--
.../patches/debian/opt/cloud/bin/update_config.py | 18 +-
5 files changed, 153 insertions(+), 138 deletions(-)
diff --git a/core/src/com/cloud/agent/resource/virtualnetwork/facade/AbstractConfigItemFacade.java b/core/src/com/cloud/agent/resource/virtualnetwork/facade/AbstractConfigItemFacade.java
index f017384..a083012 100644
--- a/core/src/com/cloud/agent/resource/virtualnetwork/facade/AbstractConfigItemFacade.java
+++ b/core/src/com/cloud/agent/resource/virtualnetwork/facade/AbstractConfigItemFacade.java
@@ -22,6 +22,8 @@ package com.cloud.agent.resource.virtualnetwork.facade;
import java.util.Hashtable;
import java.util.LinkedList;
import java.util.List;
+import java.util.UUID;
+import org.apache.log4j.Logger;
import com.cloud.agent.api.BumpUpPriorityCommand;
import com.cloud.agent.api.SetupGuestNetworkCommand;
@@ -58,6 +60,8 @@ import com.google.gson.GsonBuilder;
public abstract class AbstractConfigItemFacade {
+ private static final Logger s_logger = Logger.getLogger(AbstractConfigItemFacade.class);
+
private final static Gson gson;
private static Hashtable<Class<? extends NetworkElementCommand>, AbstractConfigItemFacade> flyweight = new Hashtable<Class<? extends NetworkElementCommand>, AbstractConfigItemFacade>();
@@ -104,13 +108,26 @@ public abstract class AbstractConfigItemFacade {
return instance;
}
+ private static String appendUuidToJsonFiles(final String filename) {
+ String remoteFileName = new String(filename);
+ if (remoteFileName.endsWith("json")) {
+ remoteFileName += "." + UUID.randomUUID().toString();
+ }
+ return remoteFileName;
+ }
+
protected List<ConfigItem> generateConfigItems(final ConfigBase configuration) {
final List<ConfigItem> cfg = new LinkedList<>();
- final ConfigItem configFile = new FileConfigItem(VRScripts.CONFIG_PERSIST_LOCATION, destinationFile, gson.toJson(configuration));
+ final String remoteFilename = appendUuidToJsonFiles(destinationFile);
+ if (s_logger.isDebugEnabled()) {
+ s_logger.debug("Transformed filename: " + destinationFile + " to: " + remoteFilename);
+ }
+
+ final ConfigItem configFile = new FileConfigItem(VRScripts.CONFIG_PERSIST_LOCATION, remoteFilename, gson.toJson(configuration));
cfg.add(configFile);
- final ConfigItem updateCommand = new ScriptConfigItem(VRScripts.UPDATE_CONFIG, destinationFile);
+ final ConfigItem updateCommand = new ScriptConfigItem(VRScripts.UPDATE_CONFIG, remoteFilename);
cfg.add(updateCommand);
return cfg;
diff --git a/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java b/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java
index 7794249..e139ecb 100644
--- a/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java
+++ b/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java
@@ -332,6 +332,9 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv
if (details == null) {
details = parser.getLines();
}
+
+ s_logger.debug("Executing script in VR: " + script);
+
return new ExecutionResult(command.getExitValue() == 0, details);
}
@@ -340,6 +343,8 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv
final File permKey = new File("/root/.ssh/id_rsa.cloud");
String error = null;
+ s_logger.debug("Creating file in VR, with ip: " + routerIp + ", file: " + filename);
+
try {
SshHelper.scpTo(routerIp, 3922, "root", permKey, null, path, content.getBytes(), filename, null);
} catch (final Exception e) {
diff --git a/systemvm/patches/debian/opt/cloud/bin/configure.py b/systemvm/patches/debian/opt/cloud/bin/configure.py
index b8a3e02..1301df0 100755
--- a/systemvm/patches/debian/opt/cloud/bin/configure.py
+++ b/systemvm/patches/debian/opt/cloud/bin/configure.py
@@ -17,33 +17,29 @@
# specific language governing permissions and limitations
# under the License.
import sys
-import os
import base64
-from merge import DataBag
-from pprint import pprint
-import subprocess
+from collections import OrderedDict
+
import logging
import re
-import time
-import shutil
+
import os.path
import os
from fcntl import flock, LOCK_EX, LOCK_UN
-from cs.CsDatabag import CsDataBag, CsCmdLine
-import cs.CsHelper
+from cs.CsDatabag import CsDataBag
from cs.CsNetfilter import CsNetfilters
from cs.CsDhcp import CsDhcp
from cs.CsRedundant import *
from cs.CsFile import CsFile
-from cs.CsApp import CsApache, CsDnsmasq
from cs.CsMonitor import CsMonitor
from cs.CsLoadBalancer import CsLoadBalancer
from cs.CsConfig import CsConfig
from cs.CsProcess import CsProcess
from cs.CsStaticRoutes import CsStaticRoutes
+OCCURRENCES = 1
class CsPassword(CsDataBag):
@@ -960,13 +956,49 @@ class CsForwardingRules(CsDataBag):
self.fw.append(["nat", "front", "-A POSTROUTING -s %s -d %s -j SNAT -o eth0 --to-source %s" % (self.getNetworkByIp(rule['internal_ip']),rule["internal_ip"], self.getGuestIp())])
+class IpTablesExecutor:
+
+ config = None
+
+ def __init__(self, config):
+ self.config = config
+
+ def process(self):
+ acls = CsAcl('networkacl', self.config)
+ acls.process()
+
+ acls = CsAcl('firewallrules', self.config)
+ acls.process()
+
+ fwd = CsForwardingRules("forwardingrules", self.config)
+ fwd.process()
+
+ vpns = CsSite2SiteVpn("site2sitevpn", self.config)
+ vpns.process()
+
+ rvpn = CsRemoteAccessVpn("remoteaccessvpn", self.config)
+ rvpn.process()
+
+ lb = CsLoadBalancer("loadbalancer", self.config)
+ lb.process()
+
+ logging.debug("Configuring iptables rules")
+ nf = CsNetfilters(True)
+ nf.compare(self.config.get_fw())
+
+ logging.debug("Configuring iptables rules done ...saving rules")
+
+ # Save iptables configuration - will be loaded on reboot by the iptables-restore that is configured on /etc/rc.local
+ CsHelper.save_iptables("iptables-save", "/etc/iptables/router_rules.v4")
+ CsHelper.save_iptables("ip6tables-save", "/etc/iptables/router_rules.v6")
+
def main(argv):
# The file we are currently processing, if it is "cmd_line.json" everything will be processed.
process_file = argv[1]
- # process_file can be None, if so assume cmd_line.json
if process_file is None:
- process_file = "cmd_line.json"
+ logging.debug("No file was received, do not go on processing the other actions. Just leave for now.")
+ return
# Track if changes need to be committed to NetFilter
iptables_change = False
@@ -977,108 +1009,60 @@ def main(argv):
logging.basicConfig(filename=config.get_logger(),
level=config.get_level(),
format=config.get_format())
- try:
- # Load stored ip adresses from disk to CsConfig()
- config.set_address()
-
- logging.debug("Configuring ip addresses")
- config.address().compare()
- config.address().process()
-
- if process_file in ["cmd_line.json", "guest_network.json"]:
- logging.debug("Configuring Guest Network")
- iptables_change = True
-
- if process_file in ["cmd_line.json", "vm_password.json"]:
- logging.debug("Configuring vmpassword")
- password = CsPassword("vmpassword", config)
- password.process()
-
- if process_file in ["cmd_line.json", "vm_metadata.json"]:
- logging.debug("Configuring vmdata")
- metadata = CsVmMetadata('vmdata', config)
- metadata.process()
-
- if process_file in ["cmd_line.json", "network_acl.json"]:
- logging.debug("Configuring networkacl")
- iptables_change = True
-
- if process_file in ["cmd_line.json", "firewall_rules.json"]:
- logging.debug("Configuring firewall rules")
- iptables_change = True
-
- if process_file in ["cmd_line.json", "forwarding_rules.json", "staticnat_rules.json"]:
- logging.debug("Configuring PF rules")
- iptables_change = True
-
- if process_file in ["cmd_line.json", "site_2_site_vpn.json"]:
- logging.debug("Configuring s2s vpn")
- iptables_change = True
-
- if process_file in ["cmd_line.json", "remote_access_vpn.json"]:
- logging.debug("Configuring remote access vpn")
- iptables_change = True
-
- if process_file in ["cmd_line.json", "vpn_user_list.json"]:
- logging.debug("Configuring vpn users list")
- vpnuser = CsVpnUser("vpnuserlist", config)
- vpnuser.process()
-
- if process_file in ["cmd_line.json", "vm_dhcp_entry.json", "dhcp.json"]:
- logging.debug("Configuring dhcp entry")
- dhcp = CsDhcp("dhcpentry", config)
- dhcp.process()
-
- if process_file in ["cmd_line.json", "load_balancer.json"]:
- logging.debug("Configuring load balancer")
- iptables_change = True
-
- if process_file in ["cmd_line.json", "monitor_service.json"]:
- logging.debug("Configuring monitor service")
- mon = CsMonitor("monitorservice", config)
- mon.process()
-
- # If iptable rules have changed, apply them.
- if iptables_change:
- acls = CsAcl('networkacl', config)
- acls.process()
-
- acls = CsAcl('firewallrules', config)
- acls.flushAllowAllEgressRules()
- acls.process()
-
- fwd = CsForwardingRules("forwardingrules", config)
- fwd.process()
-
- vpns = CsSite2SiteVpn("site2sitevpn", config)
- vpns.process()
-
- rvpn = CsRemoteAccessVpn("remoteaccessvpn", config)
- rvpn.process()
-
- lb = CsLoadBalancer("loadbalancer", config)
- lb.process()
-
- logging.debug("Configuring iptables rules")
- nf = CsNetfilters()
- nf.compare(config.get_fw())
-
- logging.debug("Configuring iptables rules done ...saving rules")
-
- # Save iptables configuration - will be loaded on reboot by the iptables-restore that is configured on /etc/rc.local
- CsHelper.save_iptables("iptables-save", "/etc/iptables/router_rules.v4")
- CsHelper.save_iptables("ip6tables-save", "/etc/iptables/router_rules.v6")
-
- red = CsRedundant(config)
- red.set()
-
- if process_file in ["cmd_line.json", "static_routes.json"]:
- logging.debug("Configuring static routes")
- static_routes = CsStaticRoutes("staticroutes", config)
- static_routes.process()
- except Exception:
- logging.exception("Exception while configuring router")
- return 1
+ # Load stored ip adresses from disk to CsConfig()
+ config.set_address()
+
+ logging.debug("Configuring ip addresses")
+ config.address().compare()
+ config.address().process()
+
+ databag_map = OrderedDict([("guest_network.json", {"process_iptables" : True, "executor" : IpTablesExecutor(config)}),
+ ("vm_password.json", {"process_iptables" : False, "executor" : CsPassword("vmpassword", config)}),
+ ("vm_metadata.json", {"process_iptables" : False, "executor" : CsVmMetadata('vmdata', config)}),
+ ("network_acl.json", {"process_iptables" : True, "executor" : IpTablesExecutor(config)}),
+ ("firewall_rules.json", {"process_iptables" : True, "executor" : IpTablesExecutor(config)}),
+ ("forwarding_rules.json", {"process_iptables" : True, "executor" : IpTablesExecutor(config)}),
+ ("staticnat_rules.json", {"process_iptables" : True, "executor" : IpTablesExecutor(config)}),
+ ("site_2_site_vpn.json", {"process_iptables" : True, "executor" : IpTablesExecutor(config)}),
+ ("remote_access_vpn.json", {"process_iptables" : True, "executor" : IpTablesExecutor(config)}),
+ ("vpn_user_list.json", {"process_iptables" : False, "executor" : CsVpnUser("vpnuserlist", config)}),
+ ("vm_dhcp_entry.json", {"process_iptables" : False, "executor" : CsDhcp("dhcpentry", config)}),
+ ("dhcp.json", {"process_iptables" : False, "executor" : CsDhcp("dhcpentry", config)}),
+ ("load_balancer.json", {"process_iptables" : True, "executor" : IpTablesExecutor(config)}),
+ ("monitor_service.json", {"process_iptables" : False, "executor" : CsMonitor("monitorservice", config)}),
+ ("static_routes.json", {"process_iptables" : False, "executor" : CsStaticRoutes("staticroutes", config)})
+ ])
+
+ if process_file.count("cmd_line.json") == OCCURRENCES:
+ logging.debug("cmd_line.json changed. All other files will be processed as well.")
+
+ while databag_map:
+ item = databag_map.popitem(last = False)
+ item_name = item[0]
+ item_dict = item[1]
+ if not item_dict["process_iptables"]:
+ executor = item_dict["executor"]
+ executor.process()
+
+ iptables_executor = IpTablesExecutor(config)
+ iptables_executor.process()
+ else:
+ while databag_map:
+ item = databag_map.popitem(last = False)
+ item_name = item[0]
+ item_dict = item[1]
+ if process_file.count(item_name) == OCCURRENCES:
+ executor = item_dict["executor"]
+ executor.process()
+
+ if item_dict["process_iptables"]:
+ iptables_executor = IpTablesExecutor(config)
+ iptables_executor.process()
+
+ break
+
+ red = CsRedundant(config)
+ red.set()
if __name__ == "__main__":
main(sys.argv)
diff --git a/systemvm/patches/debian/opt/cloud/bin/merge.py b/systemvm/patches/debian/opt/cloud/bin/merge.py
index 1c9adf2..ef20d0b 100755
--- a/systemvm/patches/debian/opt/cloud/bin/merge.py
+++ b/systemvm/patches/debian/opt/cloud/bin/merge.py
@@ -18,8 +18,10 @@
import json
import os
-import time
+import uuid
import logging
+import gzip
+import shutil
import cs_ip
import cs_guestnetwork
import cs_cmdline
@@ -36,8 +38,6 @@ import cs_remoteaccessvpn
import cs_vpnusers
import cs_staticroutes
-from pprint import pprint
-
class DataBag:
@@ -282,22 +282,26 @@ class QueueFile:
if data is not None:
self.data = data
self.type = self.data["type"]
- proc = updateDataBag(self)
+ updateDataBag(self)
return
- fn = self.configCache + '/' + self.fileName
+ filename = '{cache_location}/{json_file}'.format(cache_location = self.configCache, json_file = self.fileName)
try:
- handle = open(fn)
- except IOError:
- logging.error("Could not open %s", fn)
+ handle = open(filename)
+ except IOError as exception:
+ error_message = ("Exception occurred with the following exception error '{error}'. Could not open '{file}'. "
+ "It seems that the file has already been moved.".format(error = exception, file = filename))
+ logging.error(error_message)
else:
+ logging.info("Continuing with the processing of file '{file}'".format(file = filename))
+
self.data = json.load(handle)
self.type = self.data["type"]
handle.close()
if self.keep:
- self.__moveFile(fn, self.configCache + "/processed")
+ self.__moveFile(filename, self.configCache + "/processed")
else:
- os.remove(fn)
- proc = updateDataBag(self)
+ os.remove(filename)
+ updateDataBag(self)
def setFile(self, name):
self.fileName = name
@@ -314,8 +318,15 @@ class QueueFile:
def __moveFile(self, origPath, path):
if not os.path.exists(path):
os.makedirs(path)
- timestamp = str(int(round(time.time())))
- os.rename(origPath, path + "/" + self.fileName + "." + timestamp)
+ originalName = os.path.basename(origPath)
+ if originalName.count(".") == 1:
+ originalName += "." + str(uuid.uuid4())
+ zipped_file_name = path + "/" + originalName + ".gz"
+ with open(origPath, 'rb') as f_in, gzip.open(zipped_file_name, 'wb') as f_out:
+ shutil.copyfileobj(f_in, f_out)
+ os.remove(origPath)
+
+ logging.debug("Processed file written to %s", zipped_file_name)
class PrivateGatewayHack:
diff --git a/systemvm/patches/debian/opt/cloud/bin/update_config.py b/systemvm/patches/debian/opt/cloud/bin/update_config.py
index ab08e03..82e4320 100755
--- a/systemvm/patches/debian/opt/cloud/bin/update_config.py
+++ b/systemvm/patches/debian/opt/cloud/bin/update_config.py
@@ -24,8 +24,11 @@ from subprocess import PIPE, STDOUT
import os
import os.path
import configure
+import glob
import json
+OCCURRENCES = 1
+
logging.basicConfig(filename='/var/log/cloud.log', level=logging.INFO, format='%(asctime)s %(filename)s %(funcName)s:%(lineno)d %(message)s')
# first commandline argument should be the file to process
@@ -72,7 +75,8 @@ def is_guestnet_configured(guestnet_dict, keys):
print "[WARN] update_config.py :: Reconfiguring guest network..."
return False
- file = open(jsonCmdConfigPath)
+ fn = min(glob.iglob(jsonCmdConfigPath + '*'), key=os.path.getctime)
+ file = open(fn)
new_guestnet_dict = json.load(file)
if not new_guestnet_dict['add']:
@@ -111,19 +115,13 @@ def is_guestnet_configured(guestnet_dict, keys):
return exists
-if not (os.path.isfile(jsonCmdConfigPath) and os.access(jsonCmdConfigPath, os.R_OK)):
+filename = min(glob.iglob(jsonCmdConfigPath + '*'), key=os.path.getctime)
+if not (os.path.isfile(filename) and os.access(filename, os.R_OK)):
print "[ERROR] update_config.py :: You are telling me to process %s, but i can't access it" % jsonCmdConfigPath
sys.exit(1)
-# If the command line json file is unprocessed process it
-# This is important or, the control interfaces will get deleted!
-if os.path.isfile(jsonPath % "cmd_line.json"):
- qf = QueueFile()
- qf.setFile("cmd_line.json")
- qf.load(None)
-
# If the guest network is already configured and have the same IP, do not try to configure it again otherwise it will break
-if sys.argv[1] == "guest_network.json":
+if sys.argv[1] and sys.argv[1].count("guest_network.json") == OCCURRENCES:
if os.path.isfile(currentGuestNetConfig):
file = open(currentGuestNetConfig)
guestnet_dict = json.load(file)
--
To stop receiving notification emails like this one, please contact
"commits@cloudstack.apache.org" <co...@cloudstack.apache.org>.