You are viewing a plain text version of this content. The canonical link for it is here.
Posted to users@cloudstack.apache.org by Engelmann Florian <fl...@everyware.ch> on 2017/02/28 11:56:47 UTC

KVM VirtualRouter broken NIC deployment

Hi,

we're running into a strange error deploying a virtual router on KVM. The router ends up with 4 nics like eth0 - eth3. The agent log looks like:

2017-02-28 11:48:16,108 DEBUG [cloud.agent.Agent] (agentRequest-Handler-2:null) (logid:9e499072) Request:Seq 247-4546946773783937036:  { Cmd , MgmtId: 345049101620, via: 247, Ver: v1, Flags: 100011, [...
{
	"com.cloud.agent.api.StartCommand": {
		"vm": {
			"id": 4231,
			"name": "r-4231-VM",
			"type": "DomainRouter",
			"cpus": 1,
			"minSpeed": 125,
			"maxSpeed": 500,
			"minRam": 268435456,
			"maxRam": 268435456,
			"arch": "x86_64",
			"os": "Debian GNU/Linux 7(64-bit)",
			"platformEmulator": "Debian GNU/Linux 7(64-bit)",
			"bootArgs": " template=domP name=r-42",
			"enableHA": true,
			"limitCpuUse": false,
			"enableDynamicallyScaleVm": false,
			"vncPassword": "0pSnyIHcvO_hHkd4Lq9jNg",
			"params": {
				"memoryOvercommitRatio": "1.0",
				"cpuOvercommitRatio": "4.0"
			},
			"uuid": "ff845636-6e98-478e-b815-037b050e747e",
			"disks":
[...]
			"nics": [{
				"deviceId": 2,
				"networkRateMbps": 120,
				"defaultNic": true,
				"pxeDisable": true,
				"nicUuid": "0912c2c4-b42e-4b68-b172-4a2404165536",
				"uuid": "d76510c2-208f-4f91-a01f-39865fac7372",
				"ip": "xx7.7xx.xx2.x04",
				"netmask": "255.255.255.xxx",
				"gateway": "xx7.7xx.xxx.xxx",
				"mac": "06:7c:44:00:xx:xx",
				"dns1": "xxxxxxxxx",
				"dns2": "xxxxxxxxx",
				"broadcastType": "Vlan",
				"type": "Public",
				"broadcastUri": "vlan://2099",
				"isolationUri": "vlan://2099",
				"isSecurityGroupEnabled": false,
				"name": "cloudbr0"
			}, {
				"deviceId": 0,
				"networkRateMbps": 120,
				"defaultNic": false,
				"pxeDisable": true,
				"nicUuid": "a64c5e61-8d40-4791-9786-9d4256c11cee",
				"uuid": "f7e76386-ce2a-4d43-846a-5ee197610519",
				"ip": "172.16.0.1",
				"netmask": "255.255.255.0",
				"mac": "02:00:06:0f:00:02",
				"dns1": "8.8.8.8",
				"dns2": "8.8.4.4",
				"broadcastType": "Vlan",
				"type": "Guest",
				"broadcastUri": "vlan://3926",
				"isolationUri": "vlan://3926",
				"isSecurityGroupEnabled": false,
				"name": "cloudbr0"
			}, {
				"deviceId": 1,
				"networkRateMbps": -1,
				"defaultNic": false,
				"pxeDisable": true,
				"nicUuid": "ab3a033e-0007-4eb1-bbba-2ee25350f628",
				"uuid": "9dab4d74-c260-4c55-9dce-2018ac5b1d6e",
				"ip": "169.254.1.224",
				"netmask": "255.255.0.0",
				"gateway": "169.254.0.1",
				"mac": "0e:00:a9:fe:01:e0",
				"broadcastType": "LinkLocal",
				"type": "Control",
				"isSecurityGroupEnabled": false
			}]
		},
		"hostIp": "10.1.0.230",
		"executeInSequence": false,
		"wait": 0
	}
}, [...]
 {
	"com.cloud.agent.api.routing.IpAssocCommand": {
		"ipAddresses": [{
			"accountId": 168,
			"publicIp": "217.71.92.204",
			"sourceNat": true,
			"add": true,
			"oneToOneNat": false,
			"firstIP": true,
			"broadcastUri": "vlan://2099",
			"vlanGateway": "xx7.xxx.xx2.xxx",
			"vlanNetmask": "255.255.255.xxx",
			"vifMacAddress": "06:af:fa:00:xx:xx",
			"networkRate": 120,
			"trafficType": "Public",
			"networkName": "cloudbr0",
			"newNic": false
		}],
		"accessDetails": {
			"router.guest.ip": "172.16.0.1",
			"zone.network.type": "Advanced",
			"router.ip": "169.254.1.224",
			"router.name": "r-4231-VM"
		},
		"wait": 0
	}
}, {
	
[...]


The Router ends up with the following interfaces:

1: lo: <LOOPBACK,UP,LOWER_UP> mtu 16436 qdisc noqueue state UNKNOWN 
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP qlen 1000
    link/ether 02:00:06:0f:00:02 brd ff:ff:ff:ff:ff:ff
    inet 172.16.0.1/24 brd 172.16.0.255 scope global eth0
3: eth1: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP qlen 1000
    link/ether 0e:00:a9:fe:01:e0 brd ff:ff:ff:ff:ff:ff
    inet 169.254.1.224/16 brd 169.254.255.255 scope global eth1
4: eth2: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP qlen 1000
    link/ether 06:7c:44:00:xx:xx brd ff:ff:ff:ff:ff:ff
5: eth3: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP qlen 1000
    link/ether 06:af:fa:00:xx:xx brd ff:ff:ff:ff:ff:ff
    inet xx7.xxx.xx2.xxx/xx brd 217.71.92.223 scope global eth3


And a missconfigured NAT:

iptables -t nat -L -v
Chain PREROUTING (policy ACCEPT 86 packets, 6024 bytes)
 pkts bytes target     prot opt in     out     source               destination         

Chain INPUT (policy ACCEPT 81 packets, 5724 bytes)
 pkts bytes target     prot opt in     out     source               destination         

Chain OUTPUT (policy ACCEPT 17 packets, 1034 bytes)
 pkts bytes target     prot opt in     out     source               destination         

Chain POSTROUTING (policy ACCEPT 16 packets, 958 bytes)
 pkts bytes target     prot opt in     out     source               destination         
    1    76 SNAT       all  --  any    eth2    anywhere             anywhere             to:xx7.xxx.xx2.xxx

As you can see the SNAT is configured to use eth2 and not eth3. We are wondering why eth3 exists at all? Any hints?

After rebooting the Virtual Router the NAT IP gets bound on eth2 and everything works as expected.

All the best,
Florian