diff --git a/autocertkit/ack_cli.py b/autocertkit/ack_cli.py index 96e1df2e..3cc90167 100755 --- a/autocertkit/ack_cli.py +++ b/autocertkit/ack_cli.py @@ -298,6 +298,14 @@ def parse_static_config(configParser, section): raise utils.InvalidArgument( option, config[option], "Should not be empty!") + ip_s = utils.IPv4Addr(config['ip_start'], config['netmask'], config['gw']) + ip_s.validate() + ip_e = utils.IPv4Addr(config['ip_end'], config['netmask'], config['gw']) + ip_e.validate() + if ip_s.get_subnet_host()[1] >= ip_e.get_subnet_host()[1]: + raise utils.InvalidArgument('ip_end', config['ip_end'], + "Should be greater than 'ip_start' %s!" % config['ip_start']) + return config diff --git a/autocertkit/network_tests.py b/autocertkit/network_tests.py index 4f8febb8..758fdf66 100644 --- a/autocertkit/network_tests.py +++ b/autocertkit/network_tests.py @@ -442,7 +442,7 @@ def test_vlan_high_port(self, session): rec = {} rec['info'] = ping_result - if "0% packet loss" not in ping_result: + if " 0% packet loss" not in ping_result: raise TestCaseError("Error: Ping transmittion failed. %s" % ping_result) @@ -975,6 +975,7 @@ def test_ping(self, session): class MulticastTestClass(IperfTestClass): """ Subclass that runs multicast test""" + REQUIRED_FOR = ">= %s" % XCP_MIN_VER_WITH_MULTICAST caps = [MULTICAST_CAP] required = False diff --git a/autocertkit/testbase.py b/autocertkit/testbase.py index 38132041..c090b4c3 100644 --- a/autocertkit/testbase.py +++ b/autocertkit/testbase.py @@ -326,17 +326,20 @@ def generate_static_net_conf(self): log.debug("Create static config for %s (%s)" % (iface, vlan)) key_name = "%s_%s" % (iface, vlan) - assert key_name not in res.keys( - ), "Duplicate static IP addressing specified for %s (%s)" % (iface, vlan) + assert key_name not in res.keys(), \ + "Duplicate static IP addressing specified for %s (%s)" % ( + iface, vlan) res[key_name] = sm log.debug("Added static conf for '%s'" % key_name) mgmt = get_pool_management_device(self.session) - if 'static_management' in netconf and mgmt not in netconf: - log.debug("Create static config for mgmt device %s" % mgmt) + log.debug("The pool management device is %s" % mgmt) + if 'static_management' in netconf: + assert mgmt not in netconf, \ + "'static_management' should only be specified when management " \ + "device(%s) is not being tested for certification. " % (mgmt) + log.debug("Create static config for management device %s" % mgmt) key_name = "%s_0" % (mgmt) - assert key_name not in res.keys( - ), "'static_management' should only be specified when management device(%s) is not being tested for certification." % (iface) res[key_name] = StaticIPManager(netconf['static_management']) log.debug("Added static conf for '%s'" % key_name) diff --git a/autocertkit/tests/utils_tests.py b/autocertkit/tests/utils_tests.py index 0929e880..313e0d4c 100644 --- a/autocertkit/tests/utils_tests.py +++ b/autocertkit/tests/utils_tests.py @@ -42,86 +42,115 @@ def testGreaterThanFalse(self): self._exp_false('> 5.6 SP2', '5.6 FP1') -class StaticIPUtilsTests(unittest_base.DevTestCase): - """Verify that the class methods for manipulating - static IP addresses work correctly""" - - def _test_on_subnet(self, ip_a, ip_b, mask, expect=True): - ipa = utils.IPv4Addr(ip_a, mask, '192.168.0.1') - ipb = utils.IPv4Addr(ip_b, mask, '192.168.0.1') - - if ipa.on_subnet(ipb) and not expect: - raise Exception("'%s' and '%s' on subnet '%s' - '%s'" % (ip_a, - ip_b, - mask, - expect)) - - def _test_increment_ip(self, start, result, expect=True): - conf = {'ip_start': '192.168.0.1', - 'ip_end': '192.168.0.10', - 'netmask': '255.255.255.0', - 'gw': '192.168.0.1'} - sim = utils.StaticIPManager(conf) - - res = sim.increment_ip_string(start) - - if res != result and expect: - raise Exception("Error: '%s' incremeneted, should equal '%s' not '%s'" % - (start, result, res)) - - def testOnSubnet(self): - self._test_on_subnet('192.168.0.10', - '192.168.0.40', - '255.255.255.0') - - def testNotOnSubnet(self): - self._test_on_subnet('192.168.0.10', - '192.128.0.40', - '255.255.255.0', - expect=False) - - def testIncrementIPs(self): - self._test_increment_ip('192.168.0.1', '192.168.0.2') - self._test_increment_ip('192.168.0.1', '192.168.0.10', expect=False) - - def testEnumerateIPs(self): - conf = {'ip_start': '10.80.227.143', - 'ip_end': '10.80.227.151', - 'netmask': '255.255.0.0', - 'gw': '10.80.227.1'} - - full_list = ['10.80.227.143', '10.80.227.144', '10.80.227.145', '10.80.227.146', - '10.80.227.147', '10.80.227.148', '10.80.227.149', '10.80.227.150', - '10.80.227.151'] - - sim = utils.StaticIPManager(conf) - free_list = sim.ip_pool - - if len(free_list) != len(full_list): - raise Exception("Error: we expect there to be %d IPs, enumerate produced %d." % - (len(full_list), len(free_list))) - - for i in range(len(full_list)): - if free_list[i].addr != full_list[i]: - raise Exception("Error: Enumerate IP returns %s, we expect %s" % (free_list, - full_list)) - - def testLoanStaticIP(self): - conf = {'ip_start': '192.168.0.5', - 'ip_end': '192.168.0.10', - 'netmask': '255.255.255.0', - 'gw': '192.168.0.1'} - - sim = utils.StaticIPManager(conf) - - borrowed_ip = sim.get_ip() - assert(sim.available_ips() == 5) - assert(len(sim.in_use) == 1) - - sim.return_ip(borrowed_ip) - - assert(sim.available_ips() == 6) - assert(len(sim.in_use) == 0) +class IPv4AddrTests(unittest.TestCase): + + def test_check_ip_format(self): + utils.IPv4Addr.check_ip_format('192.168.0.1') + self.assertRaises( + Exception, lambda: utils.IPv4Addr.check_ip_format('192.168.0.256')) + self.assertRaises( + Exception, lambda: utils.IPv4Addr.check_ip_format('192.168.1')) + self.assertRaises( + Exception, lambda: utils.IPv4Addr.check_ip_format('192.168.0.0.1')) + self.assertRaises( + Exception, lambda: utils.IPv4Addr.check_ip_format('192.168.0.01')) + + def test_check_netwrok_mask(self): + utils.IPv4Addr.check_netwrok_mask('255.255.255.0') + utils.IPv4Addr.check_netwrok_mask('255.255.0.0') + utils.IPv4Addr.check_netwrok_mask('255.0.0.0') + utils.IPv4Addr.check_netwrok_mask('255.255.240.0') + self.assertRaises( + Exception, lambda: utils.IPv4Addr.check_netwrok_mask('255.255.255.255')) + self.assertRaises( + Exception, lambda: utils.IPv4Addr.check_netwrok_mask('0.0.0.0')) + + def test_check_special_ip(self): + utils.IPv4Addr.check_special_ip('192.168.0.1', '255.255.255.0') + self.assertRaises(Exception, lambda: utils.IPv4Addr.check_special_ip( + '192.168.0.0', '255.255.255.0')) + self.assertRaises(Exception, lambda: utils.IPv4Addr.check_special_ip( + '192.168.0.255', '255.255.255.0')) + + def test_split(self): + subnet, host = utils.IPv4Addr.split('192.168.0.1', '255.255.255.0') + self.assertEqual(subnet, (192 << 24) + (168 << 16) + (0 << 8)) + self.assertEqual(host, 1) + + def test_aton(self): + n_ip = utils.IPv4Addr.aton('192.168.0.1') + self.assertEqual(n_ip, (192 << 24) + (168 << 16) + (0 << 8) + 1) + self.assertRaises( + Exception, lambda: utils.IPv4Addr.aton('192.168.0.256')) + + def test_ntoa(self): + ip = utils.IPv4Addr.ntoa((192 << 24) + (168 << 16) + (0 << 8) + 1) + self.assertEqual(ip, '192.168.0.1') + self.assertRaises(Exception, lambda: utils.IPv4Addr.ntoa(0x100000000)) + + def test_validate_netmask(self): + utils.IPv4Addr.validate_netmask('255.255.255.0') + + def test_validate_ip(self): + utils.IPv4Addr.validate_ip('192.168.255.1', '255.255.255.0') + + def test_in_same_subnet(self): + utils.IPv4Addr.in_same_subnet( + '192.168.255.1', '192.168.255.254', '255.255.255.0') + + def test_validate(self): + ip = utils.IPv4Addr('192.168.0.10', '255.255.255.0', '192.168.0.1') + ip.validate() + ip = utils.IPv4Addr('192.16.254.10', '255.240.0.0', '192.16.0.1') + ip.validate() + + def test_get_subnet_host(self): + ip = utils.IPv4Addr('192.168.0.2', '255.255.255.0', '192.168.0.1') + subnet, host = ip.get_subnet_host() + self.assertEqual(subnet, (192 << 24) + (168 << 16) + (0 << 8)) + self.assertEqual(host, 2) + + +class StaticIPManagerTests(unittest.TestCase): + + def setUp(self): + self.conf = {'ip_start': '192.168.0.2', + 'ip_end': '192.168.0.5', + 'netmask': '255.255.255.0', + 'gw': '192.168.0.1'} + self.sm = utils.StaticIPManager(self.conf) + + def tearDown(self): + self.sm.release_all() + + def test_get_ip(self): + ip = self.sm.get_ip() + self.assertEqual(ip.addr, '192.168.0.2') + self.assertEqual(ip.netmask, '255.255.255.0') + self.assertEqual(ip.gateway, '192.168.0.1') + ip = self.sm.get_ip() + self.assertEqual(ip.addr, '192.168.0.3') + ip = self.sm.get_ip() + self.assertEqual(ip.addr, '192.168.0.4') + ip = self.sm.get_ip() + self.assertEqual(ip.addr, '192.168.0.5') + self.assertRaises(Exception, lambda: self.sm.get_ip()) + + self.sm.release_all() + + def test_return_ip(self): + free1 = self.sm.available_ips() + ip = self.sm.get_ip() + free2 = self.sm.available_ips() + self.assertEqual(free1 - 1, free2) + + self.sm.return_ip(ip) + free3 = self.sm.available_ips() + self.assertEqual(free1, free3) + + self.assertRaises(Exception, lambda: self.sm.return_ip(ip)) + + self.sm.release_all() class ValueInRangeFunctions(unittest.TestCase): diff --git a/autocertkit/utils.py b/autocertkit/utils.py index b50ec6f8..bc88aea0 100644 --- a/autocertkit/utils.py +++ b/autocertkit/utils.py @@ -36,18 +36,17 @@ import sys import time import ssh -from xml.dom import minidom -import tarfile import signal from datetime import datetime import os -import base64 import threading import re import json import binascii -import uuid +import socket +import struct +import ctypes from acktools.net import route, generate_mac import acktools.log @@ -70,6 +69,9 @@ MULTICAST_CAP = "MULTICAST" SRIOV_CAP = "SR-IOV" +# XCP minimum version with Multicast support +XCP_MIN_VER_WITH_MULTICAST = "2.3.0" # XenServer 7.2 + # XCP minimum version with SR-IOV support XCP_MIN_VER_WITH_SRIOV = "2.6.0" # XenServer 7.5 @@ -205,84 +207,96 @@ def __init__(self, ip, netmask, gateway): self.addr = ip self.netmask = netmask self.gateway = gateway - self.validate_ip() - self.validate_netmask() - - def validate_ip(self): - arr = self.addr.split('.') - if len(arr) != 4: - raise Exception("Error: IP address is not correct: '%s'" % - self.addr) - - for i in range(0, 4): - if int(arr[i]) > 255: - raise Exception("IP address is out of range: %s" % self.addr) - - def validate_netmask(self): - arr = self.netmask.split('.') - - # Extract netmask into a single mask - mask_str = "" - for group in arr: - mask_str = mask_str + int_to_bin(int(group))[2:] - - # For a netmask, we expect a contigious line of ones. - zero_pos = mask_str.find('0') - - if zero_pos == -1: - return True - else: - if '1' in mask_str[zero_pos + 1:]: - raise Exception("Invalid netmask: '%s' ('%s')" % - (self.netmask, mask_str)) - - def to_bin(self, integer): - """Convert an integer to a 8bit string""" - if integer > 256: - raise Exception("'to_bin' method is only for 8bit integers") - - bin_str = int_to_bin(integer)[2:] - - tmp_str = "" - for i in range(8 - len(bin_str)): - tmp_str = tmp_str + "0" - return tmp_str + bin_str + @staticmethod + def check_ip_format(ip): + # check by socket which can cover most of cases + IPv4Addr.aton(ip) - def byte_mask_match(self, bina, binb, mask): - """For two 8bit binary strings, check that they match for - any masked bits""" + if len(ip.split('.')) != 4: + raise Exception( + "The IP address %s is invalid with number of '.' is not 3" + % ip) - for i in range(8): - if mask[i] == '1': - if bina[i] != binb[i]: - return False - elif mask[i] == '0': - continue - else: + # check no prefix 0 in each byte part, e.g. 10.71.05.88 + for b in ip.split('.'): + if len(b) > 1 and b[0] == '0': raise Exception( - "Unexpected characted '%s' in binary string." % mask[i]) + "The IP address %s is invalid with redundant digit '0'" + % ip) + + @staticmethod + def check_netwrok_mask(mask): + n_m = IPv4Addr.aton(mask) + bs_m = "{0:0>32b}".format(n_m) + # check not interlaced (just 1s on the left and 0s on the right) + # and not all 0, and not all 1 + if bs_m.lstrip('1').rstrip('0') or bs_m[0] != '1' or bs_m[-1] != '0': + raise Exception( + "The network mask %s (%s) is invalid" % (mask, bs_m)) + + @staticmethod + def check_special_ip(ip, mask): + n_ip = IPv4Addr.aton(ip) + n_m = IPv4Addr.aton(mask) + n_mc = ctypes.c_uint(~n_m).value + # check network ip with host part is all 0 + if n_mc & n_ip == 0: + raise Exception( + "The IP address %s (%s) is network ip" % (ip, mask)) + # check broadcast ip with host part is all 1 + if n_mc & n_ip == n_mc: + raise Exception( + "The IP address %s (%s) is broadcast ip" % (ip, mask)) + + @staticmethod + def split(ip, mask): + n_ip = IPv4Addr.aton(ip) + n_m = IPv4Addr.aton(mask) + subnet = ctypes.c_uint(n_ip & n_m).value + host = ctypes.c_uint(n_ip & ~n_m).value + return (subnet, host) + + @staticmethod + def aton(ip): + try: + return struct.unpack("!I", socket.inet_aton(ip))[0] + except Exception, e: + raise Exception( + "The IP address %s is invalid, exception: %s" + % (ip, str(e))) - return True + @staticmethod + def ntoa(n_ip): + try: + return socket.inet_ntoa(struct.pack('!I', n_ip)) + except Exception, e: + raise Exception( + "The IP address 0x%x is invalid, exception: %s" + % (n_ip, str(e))) - def on_subnet(self, ip): - # Check that mask is the same - if self.netmask != ip.netmask: - return False + @staticmethod + def validate_netmask(mask): + IPv4Addr.check_ip_format(mask) + IPv4Addr.check_netwrok_mask(mask) - # Mask both IPs and check whether they are equal. - arrA = self.addr.split('.') - arrB = ip.addr.split('.') - arrMask = self.netmask.split('.') + @staticmethod + def validate_ip(ip, mask): + IPv4Addr.check_ip_format(ip) + IPv4Addr.check_special_ip(ip, mask) - for i in range(0, 4): - a = self.to_bin(int(arrA[i])) - b = self.to_bin(int(arrB[i])) - m = self.to_bin(int(arrMask[i])) + @staticmethod + def in_same_subnet(ip1, ip2, mask): + return IPv4Addr.split(ip1, mask)[0] == IPv4Addr.split(ip2, mask)[0] - if not self.byte_mask_match(a, b, m): - return False - return True + def validate(self): + IPv4Addr.validate_netmask(self.netmask) + IPv4Addr.validate_ip(self.addr, self.netmask) + IPv4Addr.validate_ip(self.gateway, self.netmask) + assert(IPv4Addr.in_same_subnet(self.addr, self.gateway, self.netmask)) + + def get_subnet_host(self): + return IPv4Addr.split(self.addr, self.netmask) def get_network_routes(session, host_ref, retry=6): @@ -312,133 +326,66 @@ class StaticIPManager(object): the caller. Allows us to do simple 'leasing' operations""" def __init__(self, conf): - # Populate the internal list of IPs - free = [] - for ip_addr in self.generate_ip_list(conf['ip_start'], - conf['ip_end']): - free.append(IPv4Addr(ip_addr, - conf['netmask'], - conf['gw'])) - - self.ip_pool = free # All the list of IPs - self.in_use = [] # Index list of used IP from ip_pool. - self.last_used = -1 # Index of IP lastly picked. Next will be 0. - self.total_ips = len(free) - - def generate_ip_list(self, ip_start, ip_end): - """Take an IP address start, and end, and compose a list of all - the IP addresses inbetween. E.g. '192.168.0.1' - '192.168.0.4' would - return ['192.168.0.1', '192.168.0.2', '192.168.0.3', '192.168.0.4'].""" - - def validate_ip(str_ip): - try: - arr = str_ip.split('.') - res = [] - for i in range(0, 4): - res.append(int(arr[i])) - if res[i] > 254: - raise Exception("Invalid IP %s" % str_ip) - return arr - except Exception, e: - raise Exception( - "Error: '%s' is not a valid IPv4 Addr (%s)" % (str_ip, str(e))) - - arr1 = validate_ip(ip_start) - arr2 = validate_ip(ip_end) - - for i in range(4): - if int(arr2[i]) < int(arr1[i]): - raise Exception("IP start ('%s') must be smaller than IP end ('%s') (%s < %s)" % - (ip_start, - ip_end, arr2[i], arr1[i])) - - res = [] - - res.append(ip_start) - - if ip_end == ip_start: - return res - - tmp_string = self.increment_ip_string(ip_start) + self.conf = conf + self.subnet, self.host_start = IPv4Addr.split( + conf['ip_start'], conf['netmask']) + _, self.host_end = IPv4Addr.split(conf['ip_end'], conf['netmask']) + self.total_ips = self.host_end - self.host_start + 1 + self._reset() + + def _reset(self): + self.ips = [False] * self.total_ips + self.free_ips = self.total_ips + self.last_used = -1 - while tmp_string != ip_end: - res.append(tmp_string) - tmp_string = self.increment_ip_string(tmp_string) + def get_ip(self): + """Return an unused IP object (if one exists)""" + id = self._get_free() + ip = IPv4Addr.ntoa(self.subnet + self.host_start + id) + return IPv4Addr(ip, self.conf['netmask'], self.conf['gw']) - # After exit, we must also add the last value - res.append(ip_end) + def return_ip(self, ip): + """For a given IP object, put it back into circulation for others to use""" + self._put_free(ip.get_subnet_host()[1] - self.host_start) - return res + def _is_free(self, id): + return not self.ips[id] - def increment_ip_string(self, string): - chars = string.split('.') - arr = [] - for i in range(4): - arr.append(int(chars[i])) + def _set_free(self, id): + self.ips[id] = False + self.free_ips += 1 - def carry(x): - if int(x) == 254: - return True - else: - return False - - if not carry(arr[3]): - arr[3] = arr[3] + 1 - elif not carry(arr[2]): - arr[3] = 1 - arr[2] = arr[2] + 1 - elif not carry(arr[1]): - arr[3] = 1 - arr[2] = 1 - arr[1] = arr[1] + 1 - elif not carry(arr[0]): - arr[3] = 1 - arr[2] = 1 - arr[1] = 1 - arr[0] = arr[0] + 1 - - if arr[0] == 255: - raise Exception("Error: Invalid to increment: %s" % string) - - return "%s.%s.%s.%s" % (arr[0], arr[1], arr[2], arr[3]) + def _set_used(self, id): + self.ips[id] = True + self.free_ips -= 1 - def get_ip(self): - """Return an unused IP object (if one exists)""" - if len(self.in_use) >= self.total_ips: - raise Exception("Error: no more IP addresses to allocate! (%d in use)" % - len(self.in_use)) - - index = (self.last_used + 1) % self.total_ips - while True: - if not index in self.in_use: - self.last_used = index - self.in_use.append(index) - return self.ip_pool[index] - index = (index + 1) % self.total_ips + def _get_next_of(self, id): + return (id + 1) % self.total_ips - def return_ip(self, ip): - """For a given IP object, attempt to remove from the 'in_use' list, and put - it back into circulation for others to use""" - if not ip in self.ip_pool: - log.debug("IP(%s) does not exist in IP pool." % (ip,)) + def _get_free(self): + while self.free_ips > 0: + self.last_used = self._get_next_of(self.last_used) + if self._is_free(self.last_used): + self._set_used(self.last_used) + return self.last_used + else: raise Exception( - "Trying to return an IP address that did not orginally exist!") + "Error: no more IP addresses to allocate! (%d in use)" % self.total_ips) - index = self.ip_pool.index(ip) - if index in self.in_use: - log.debug("IP %s is in use. Removing..." % (ip,)) - self.in_use.remove(index) - else: - log.debug("IP %s is not in use. Passing." % (ip,)) + def _put_free(self, id): + if self._is_free(id): + raise Exception( + "Error: Should not free a unused IP address id: %d" % id) + self._set_free(id) def release_all(self): """Return all of the IP addresses that are currently in use""" log.debug("Clearing in-use IP list.") - self.in_use = [] + self._reset() def available_ips(self): """Return number of unused IP in IP pool""" - return self.total_ips - len(self.in_use) + return self.free_ips class IfaceStats(object): diff --git a/plugins/autocertkit b/plugins/autocertkit index d169a313..4bea868f 100755 --- a/plugins/autocertkit +++ b/plugins/autocertkit @@ -137,6 +137,8 @@ OFFLOAD_FLAGS_OFFSET = {'sg': 1, 'ntuple': 1 << 27, 'rxhash': 1 << 28} +XCP_MIN_VER_WITH_SRIOV = "2.6.0" + # Logging setup def configure_logging(): @@ -1811,7 +1813,8 @@ def _get_local_device_bridge(session, device): host = session.xenapi.session.get_this_host(session.handle) for pif in session.xenapi.host.get_PIFs(host): # skip sriov logical pif - if not session.xenapi.PIF.get_physical(pif) and \ + if has_sriov_feature(session) and not \ + session.xenapi.PIF.get_physical(pif) and \ session.xenapi.PIF.get_sriov_logical_PIF_of(pif): continue @@ -2006,6 +2009,15 @@ def start_iperf_server(session, args): ############# Utility function ##################### +@log_exceptions +def get_xcp_version(session): + """Return the XCP version (using the master host)""" + pool = session.xenapi.pool.get_all()[0] + master_ref = session.xenapi.pool.get_master(pool) + software = session.xenapi.host.get_software_version(master_ref) + return software['platform_version'] + + @log_exceptions def get_kernel_version(session, args): """Check kernel version using uname.""" @@ -2026,6 +2038,11 @@ def get_ack_version(session, args): raise Exception("Unable to parse ack version") +@log_exceptions +def has_sriov_feature(session): + return get_xcp_version(session) >= XCP_MIN_VER_WITH_SRIOV + + @log_exceptions def get_dmidecode_output(session, args): """Returns output of dmidecode call"""