#! /usr/bin/env python # pylint: disable=missing-module-docstring,missing-function-docstring,missing-class-docstring,too-many-lines import sys import os import json import glob import logging import time import argparse import subprocess # The PCI base class for all devices network_class = { "Class": "02", "Vendor": None, "Device": None, "SVendor": None, "SDevice": None, } acceleration_class = { "Class": "12", "Vendor": None, "Device": None, "SVendor": None, "SDevice": None, } ifpga_class = { "Class": "12", "Vendor": "8086", "Device": "0b30", "SVendor": None, "SDevice": None, } encryption_class = { "Class": "10", "Vendor": None, "Device": None, "SVendor": None, "SDevice": None, } intel_processor_class = { "Class": "0b", "Vendor": "8086", "Device": None, "SVendor": None, "SDevice": None, } cavium_sso = { "Class": "08", "Vendor": "177d", "Device": "a04b,a04d", "SVendor": None, "SDevice": None, } cavium_fpa = { "Class": "08", "Vendor": "177d", "Device": "a053", "SVendor": None, "SDevice": None, } cavium_pkx = { "Class": "08", "Vendor": "177d", "Device": "a0dd,a049", "SVendor": None, "SDevice": None, } cavium_tim = { "Class": "08", "Vendor": "177d", "Device": "a051", "SVendor": None, "SDevice": None, } cavium_zip = { "Class": "12", "Vendor": "177d", "Device": "a037", "SVendor": None, "SDevice": None, } avp_vnic = { "Class": "05", "Vendor": "1af4", "Device": "1110", "SVendor": None, "SDevice": None, } octeontx2_sso = { "Class": "08", "Vendor": "177d", "Device": "a0f9,a0fa", "SVendor": None, "SDevice": None, } octeontx2_npa = { "Class": "08", "Vendor": "177d", "Device": "a0fb,a0fc", "SVendor": None, "SDevice": None, } octeontx2_dma = { "Class": "08", "Vendor": "177d", "Device": "a081", "SVendor": None, "SDevice": None, } intel_ioat_bdw = { "Class": "08", "Vendor": "8086", "Device": "6f20,6f21,6f22,6f23,6f24,6f25,6f26,6f27,6f2e,6f2f", "SVendor": None, "SDevice": None, } intel_ioat_skx = { "Class": "08", "Vendor": "8086", "Device": "2021", "SVendor": None, "SDevice": None, } intel_ntb_skx = { "Class": "06", "Vendor": "8086", "Device": "201c", "SVendor": None, "SDevice": None, } network_devices = [network_class, cavium_pkx, avp_vnic, ifpga_class] baseband_devices = [acceleration_class] crypto_devices = [encryption_class, intel_processor_class] eventdev_devices = [cavium_sso, cavium_tim, octeontx2_sso] mempool_devices = [cavium_fpa, octeontx2_npa] compress_devices = [cavium_zip] misc_devices = [intel_ioat_bdw, intel_ioat_skx, intel_ntb_skx, octeontx2_dma] logger = logging.getLogger() # global dict ethernet devices present. Dictionary indexed by PCI address. # Each device within this is itself a dictionary of device properties _DEVICES = {} # pylint:disable=invalid-name # list of supported DPDK drivers _DPDK_DRIVERS = ["igb_uio", "vfio-pci", "uio_pci_generic"] # setting file _SETTINGS_FILE = "setup_dpdk_settings.dat" # IP tools ("ip" or "ifconfig") _IP_TOOLS = None def usage(): return """ This tool helps setting up DPDK on your machine. It's based on the `dpdk-devbind` tool that is shipped with DPDK: but extends its capabilities. It takes care of everything that is needed to run a DPDK application: - Allocate hugepages used by DPDK for packet buffer allocation - Insert DPDK kernel modules - [Optional] Install the DPDK KNI module (https://doc.dpdk.org/guides/prog_guide/kernel_nic_interface.html) - Bind devices from their "standard" drivers to DPDK-controlled drivers In addition this tool allows checking the status of the devices currently bound/unbound to DPDK (similar to `dpdk-devbind --status`). Last, it allows unsetting all DPDK configuration and restore the system to its previous state. The tool has 3 available options: setup ----- Setup DPDK on your machine. This option expects as input several parameters such as the amount of hugepages to allocate, a list of (one or more) network interfaces to allocate for DPDK, the DPDK driver to use and more. After setup is complete the previous configuration is saved in the file `{settings_file}` which can later be used to restore the settings by the "restore" command. status ------ Show the status of all network devices and for each device is shows whether it's used by DPDK driver or by a kernel driver (similar to `dpdk-devbind --status`). restore ------- Restore the system to its previous state. This command assumes that the file `{settings_file}` exists and contains the information that was written by the "setup" command of the system configuration before DPDK setup. Please note that a machine restart will usually reset these configurations anyway (unless otherwise configured by the user). However this command is useful for restoring the system without a restart. Examples: ========= [Setup] allocate 512 hugepages and bind interfaces `eth0` and `eth1` to DPDK: sudo python setup_dpdk.py setup -g 512 -i eth0 eth1 [Setup] allocate 64 hugepages, bind `eth0` to DPDK and use the `uio_pci_generic` driver (the default if not specified is `igb_uio`): sudo python setup_dpdk.py setup -g 64 -i eth0 -m uio_pci_generic [Setup] allocate 1024 hugepages, bind `enp0s3` to DPDK and load the DPDK KNI driver (not loaded by default): sudo python setup_dpdk.py setup -g 1024 -i enp0s3 -k [Status] sudo python setup_dpdk.py status [Restore] sudo python setup_dpdk.py restore\n\n """.format( settings_file=_SETTINGS_FILE ) # This is roughly compatible with check_output function in subprocess module # which is only available in python 2.7. def check_output(args, stderr=None): """Run a command and capture its output""" logger.debug("running command: '%s'", " ".join(args)) return subprocess.Popen(args, stdout=subprocess.PIPE, stderr=stderr).communicate()[ 0 ] def has_driver(dev_id): """return true if a device is assigned to a driver. False otherwise""" return "Driver_str" in _DEVICES[dev_id] def get_pci_device_details(dev_id, probe_lspci): """This function gets additional details for a PCI device""" device = {} if probe_lspci: extra_info = check_output(["lspci", "-vmmks", dev_id]).splitlines() # parse lspci details for line in extra_info: if len(line) == 0: continue name, value = line.decode().split("\t", 1) name = name.strip(":") + "_str" device[name] = value # check for a unix interface name device["Interface"] = "" for base, dirs, _ in os.walk("/sys/bus/pci/devices/%s/" % dev_id): if "net" in dirs: device["Interface"] = ",".join(os.listdir(os.path.join(base, "net"))) break # check if a port is used for ssh connection device["Ssh_if"] = False device["Active"] = "" return device def clear_data(): """This function clears any old data""" global _DEVICES # pylint:disable=global-statement _DEVICES = {} def get_device_details(devices_type): """This function populates the "_DEVICES" dictionary. The keys used are the pci addresses (domain:bus:slot.func). The values are themselves dictionaries - one for each NIC.""" # pylint:disable=too-many-branches global _DEVICES # pylint:disable=global-statement # first loop through and read details for all devices # request machine readable format, with numeric IDs and String dev = {} dev_lines = check_output(["lspci", "-Dvmmnnk"]).splitlines() for dev_line in dev_lines: if len(dev_line) == 0: if device_type_match(dev, devices_type): # Replace "Driver" with "Driver_str" to have consistency of # of dictionary key names if "Driver" in dev.keys(): dev["Driver_str"] = dev.pop("Driver") if "Module" in dev.keys(): dev["Module_str"] = dev.pop("Module") # use dict to make copy of dev _DEVICES[dev["Slot"]] = dict(dev) # Clear previous device's data dev = {} else: name, value = dev_line.decode().split("\t", 1) value_list = value.rsplit(" ", 1) if len(value_list) > 1: # String stored in _str dev[name.rstrip(":") + "_str"] = value_list[0] # Numeric IDs dev[name.rstrip(":")] = ( value_list[len(value_list) - 1].rstrip("]").lstrip("[") ) if devices_type == network_devices: # check what is the interface if any for an ssh connection if # any to this host, so we can mark it later. ssh_if = [] route = check_output(["ip", "-o", "route"]) # filter out all lines for 169.254 routes route = "\n".join( filter(lambda ln: not ln.startswith("169.254"), route.decode().splitlines()) ) rt_info = route.split() for i in range(len(rt_info) - 1): if rt_info[i] == "dev": ssh_if.append(rt_info[i + 1]) # based on the basic info, get extended text details for dev in _DEVICES: if not device_type_match(_DEVICES[dev], devices_type): continue # get additional info and add it to existing data _DEVICES[dev] = _DEVICES[dev].copy() # No need to probe lspci _DEVICES[dev].update(get_pci_device_details(dev, False).items()) if devices_type == network_devices: for _if in ssh_if: if _if in _DEVICES[dev]["Interface"].split(","): _DEVICES[dev]["Ssh_if"] = True _DEVICES[dev]["Active"] = "*Active*" break # add igb_uio to list of supporting modules if needed if "Module_str" in _DEVICES[dev]: for driver in _DPDK_DRIVERS: if driver not in _DEVICES[dev]["Module_str"]: _DEVICES[dev]["Module_str"] = ( _DEVICES[dev]["Module_str"] + ",%s" % driver ) else: _DEVICES[dev]["Module_str"] = ",".join(_DPDK_DRIVERS) # make sure the driver and module strings do not have any duplicates if has_driver(dev): modules = _DEVICES[dev]["Module_str"].split(",") if _DEVICES[dev]["Driver_str"] in modules: modules.remove(_DEVICES[dev]["Driver_str"]) _DEVICES[dev]["Module_str"] = ",".join(modules) def device_type_match(dev, devices_type): # pylint:disable=too-many-nested-blocks for _, device_type in enumerate(devices_type): param_count = len([x for x in device_type.values() if x is not None]) match_count = 0 if dev["Class"][0:2] == device_type["Class"]: match_count = match_count + 1 for key in device_type: if key != "Class" and device_type[key]: value_list = device_type[key].split(",") for value in value_list: if value.strip(" ") == dev[key]: match_count = match_count + 1 # count must be the number of non None parameters to match if match_count == param_count: return True return False def dev_id_from_dev_name(dev_name): """Take a device "name" - a string passed in by user to identify a NIC device, and determine the device id - i.e. the domain:bus:slot.func - for it, which can then be used to index into the devices array""" # check if it's already a suitable index if dev_name in _DEVICES: return dev_name # check if it's an index just missing the domain part if "0000:" + dev_name in _DEVICES: return "0000:" + dev_name # check if it's an interface name, e.g. eth1 for dev in _DEVICES: if dev_name in _DEVICES[dev]["Interface"].split(","): return _DEVICES[dev]["Slot"] # if nothing else matches - error raise ValueError( "Unknown device: %s. " 'Please specify device in "bus:slot.func" format' % dev_name ) def unbind_one(dev_id, quiet, force): def handle_error(error_msg): if not quiet: logger.error(error_msg) # Unbind the device identified by "dev_id" from its current driver dev = _DEVICES[dev_id] if not has_driver(dev_id): handle_error( "notice: %s %s %s is not currently managed by any driver" % (dev["Slot"], dev["Device_str"], dev["Interface"]) ) return # prevent us disconnecting ourselves if dev["Ssh_if"] and not force: handle_error( "routing table indicates that interface %s is active. " "Skipping unbind" % dev_id ) return # write to /sys to unbind filename = "/sys/bus/pci/drivers/%s/unbind" % dev["Driver_str"] try: file_d = open(filename, "a") except: # pylint:disable=bare-except handle_error( "Error: unbind failed for %s - Cannot open %s" % (dev_id, filename) ) file_d.write(dev_id) logger.debug("unbind_one: write '%s' to '%s'", dev_id, filename) file_d.close() def bind_one(dev_id, driver, quiet, force): """Bind the device given by "dev_id" to the driver "driver". If the device is already bound to a different driver, it will be unbound first""" # pylint:disable=too-many-return-statements,too-many-branches,too-many-statements def handle_error(error_msg): if not quiet: logger.error(error_msg) dev = _DEVICES[dev_id] saved_driver = None # used to rollback any unbind in case of failure # prevent disconnection of our ssh session if dev["Ssh_if"] and not force: handle_error( "Warning: routing table indicates that interface %s is active. Not modifying" % dev_id ) return None # unbind any existing drivers we don't want if has_driver(dev_id): if dev["Driver_str"] == driver: handle_error( "Notice: %s already bound to driver %s, skipping" % (dev_id, driver) ) return None saved_driver = dev["Driver_str"] unbind_one(dev_id, quiet, force) dev.pop("Driver_str") # pop driver string # For kernels >= 3.15 driver_override can be used to specify the driver # for a device rather than relying on the driver to provide a positive # match of the device. The existing process of looking up # the vendor and device ID, adding them to the driver new_id, # will erroneously bind other devices too which has the additional burden # of unbinding those devices filename = "/sys/bus/pci/devices/%s/driver_override" % dev_id if os.path.exists(filename): try: file_d = open(filename, "w") except: # pylint:disable=bare-except handle_error( "Error: bind failed for %s - Cannot open %s" % (dev_id, filename) ) return None try: file_d.write("%s" % driver) logger.debug("bind_one: write '%s' to '%s'", driver, filename) file_d.close() except: # pylint:disable=bare-except handle_error( "Error: bind failed for %s - Cannot write driver %s to " "PCI ID " % (dev_id, driver) ) return None # For kernels < 3.15 use new_id to add PCI id's to the driver else: filename = "/sys/bus/pci/drivers/%s/new_id" % driver try: file_d = open(filename, "w") except: # pylint:disable=bare-except handle_error( "Error: bind failed for %s - Cannot open %s" % (dev_id, filename) ) return None try: # Convert Device and Vendor Id to int to write to new_id file_d.write("%04x %04x" % (int(dev["Vendor"], 16), int(dev["Device"], 16))) file_d.close() except: # pylint:disable=bare-except handle_error( "Error: bind failed for %s - Cannot write new PCI ID to " "driver %s" % (dev_id, driver) ) return None # do the bind by writing to /sys filename = "/sys/bus/pci/drivers/%s/bind" % driver try: file_d = open(filename, "a") except: # pylint:disable=bare-except logger.error("Error: bind failed for %s - Cannot open %s", dev_id, filename) if saved_driver is not None: # restore any previous driver bind_one(dev_id, saved_driver, quiet, force) return None try: file_d.write(dev_id) logger.debug("bind_one: write '%s' to '%s'", dev_id, filename) file_d.close() except: # pylint:disable=bare-except # for some reason, closing dev_id after adding a new PCI ID to new_id # results in IOError. however, if the device was successfully bound, # we don't care for any errors and can safely ignore IOError tmp = get_pci_device_details(dev_id, True) if "Driver_str" in tmp and tmp["Driver_str"] == driver: return saved_driver logger.error( "Error: bind failed for %s - Cannot bind to driver %s", dev_id, driver ) if saved_driver is not None: # restore any previous driver bind_one(dev_id, saved_driver, quiet, force) return None # For kernels > 3.15 driver_override is used to bind a device to a driver. # Before unbinding it, overwrite driver_override with empty string so that # the device can be bound to any other driver filename = "/sys/bus/pci/devices/%s/driver_override" % dev_id if os.path.exists(filename): try: file_d = open(filename, "w") except: # pylint:disable=bare-except handle_error( "Error: unbind failed for %s - Cannot open %s" % (dev_id, filename) ) return None try: file_d.write("\00") logger.debug("bind_one: write '\00' to '%s'", filename) file_d.close() except: # pylint:disable=bare-except handle_error( "Error: unbind failed for %s - Cannot open %s" % (dev_id, filename) ) return None return saved_driver def display_devices(title, dev_list, extra_params=None): """Displays to the user the details of a list of devices given in "dev_list". The "extra_params" parameter, if given, should contain a string with %()s fields in it for replacement by the named fields in each device's dictionary.""" strings = [] # this holds the strings to print. We sort before printing print("\n%s" % title) print("=" * len(title)) if len(dev_list) == 0: strings.append("") else: for dev in dev_list: if extra_params is not None: strings.append( "%s '%s %s' %s" % ( dev["Slot"], dev["Device_str"], dev["Device"], extra_params % dev, ) ) else: strings.append("%s '%s'" % (dev["Slot"], dev["Device_str"])) # sort before printing, so that the entries appear in PCI order strings.sort() print("\n".join(strings)) # print one per line def show_device_status(devices_type, device_name): kernel_drv = [] dpdk_drv = [] no_drv = [] # split our list of network devices into the three categories above for dev in _DEVICES: if device_type_match(_DEVICES[dev], devices_type): if not has_driver(dev): no_drv.append(_DEVICES[dev]) continue if _DEVICES[dev]["Driver_str"] in _DPDK_DRIVERS: dpdk_drv.append(_DEVICES[dev]) else: kernel_drv.append(_DEVICES[dev]) n_devs = len(dpdk_drv) + len(kernel_drv) + len(no_drv) # don't bother displaying anything if there are no devices if n_devs == 0: msg = "No '%s' devices detected" % device_name print("") print(msg) print("".join("=" * len(msg))) return # print each category separately, so we can clearly see what's used by DPDK if len(dpdk_drv) != 0: display_devices( "%s devices using DPDK-compatible driver" % device_name, dpdk_drv, "drv=%(Driver_str)s unused=%(Module_str)s", ) if len(kernel_drv) != 0: display_devices( "%s devices using kernel driver" % device_name, kernel_drv, "if=%(Interface)s drv=%(Driver_str)s " "unused=%(Module_str)s %(Active)s", ) if len(no_drv) != 0: display_devices( "Other %s devices" % device_name, no_drv, "unused=%(Module_str)s" ) def show_status(status_dev): """Function called when the script is passed the "--status" option. Displays to the user what devices are bound to the igb_uio driver, the kernel driver or to no driver""" if status_dev in ("net", "all"): show_device_status(network_devices, "Network") if status_dev in ("baseband", "all"): show_device_status(baseband_devices, "Baseband") if status_dev in ("crypto", "all"): show_device_status(crypto_devices, "Crypto") if status_dev in ("event", "all"): show_device_status(eventdev_devices, "Eventdev") if status_dev in ("mempool", "all"): show_device_status(mempool_devices, "Mempool") if status_dev in ("compress", "all"): show_device_status(compress_devices, "Compress") if status_dev in ("misc", "all"): show_device_status(misc_devices, "Misc (rawdev)") def load_device_data(): clear_data() get_device_details(network_devices) get_device_details(baseband_devices) get_device_details(crypto_devices) get_device_details(eventdev_devices) get_device_details(mempool_devices) get_device_details(compress_devices) get_device_details(misc_devices) def is_mount(mount_path): with open("/proc/mounts", "r") as mounts: for line in mounts: if mount_path in line.split(): return True return False def find_and_set_rte_sdk(args, settings): if args.rte_sdk: settings.rte_sdk = args.rte_sdk return if os.environ.get("RTE_SDK"): settings.rte_sdk = os.environ.get("RTE_SDK") return if hasattr(settings, "rte_sdk") and settings.rte_sdk: return raise RuntimeError( "Cannot find RTE_SDK. Searched in the settings file, OS env variables and command-line arguments" ) def verify_rte_sdk(args, settings): find_and_set_rte_sdk(args, settings) if not os.path.exists(settings.rte_sdk): raise FileExistsError("RTE_SDK path: '%s' does not exist" % settings.rte_sdk) def check_huge_pages(): with open("/proc/meminfo", "r") as meminfo: for line in meminfo: if line.startswith("HugePages_Total"): return line.split()[1] return 0 def setup_huge_pages(amount): if check_huge_pages() == amount: return nr_hugepages_path = "/sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages" with open(nr_hugepages_path, "w") as nr_huge: nr_huge.write(str(amount)) mount_dir = "/mnt/huge" if not is_mount(mount_dir): check_output(["mount", "-t", "hugetlbfs", "nodev", mount_dir]) logger.info("set up hugepages to %s", amount) def restore_huge_pages(): if check_huge_pages() == 0: return nr_hugepages_path = "/sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages" with open(nr_hugepages_path, "w") as nr_huge: nr_huge.write("0") mount_dir = "/mnt/huge" if is_mount(mount_dir): check_output(["umount", mount_dir]) logger.info("removed hugepages") def is_module_loaded(module): for line in check_output(["lsmod"]).decode("utf-8").splitlines(): if module in line.split(): return True return False def load_dpdk_module(module, settings): if is_module_loaded(module): logger.info("module '%s' already loaded", module) return if module in ["uio_pci_generic", "vfio-pci"]: output = check_output(["modprobe", module], stderr=subprocess.STDOUT) if output: raise RuntimeError( "Something went wrong with adding '%s' kernel module: %s" % (module, output.decode("utf-8")) ) settings.dpdk_module = module logger.info("loaded kernel module '%s'", module) elif module == "igb_uio": if not hasattr(settings, "rte_sdk") or not settings.rte_sdk: raise RuntimeError("Cannot find RTE_SDK value in '%s'" % _SETTINGS_FILE) output = check_output(["modprobe", "uio"], stderr=subprocess.STDOUT) if output: raise RuntimeError( "Something went wrong with adding 'uio' kernel module: %s" % output.decode("utf-8") ) logger.info("loaded kernel module 'uio'") output = check_output( ["insmod", "%s/build/kmod/igb_uio.ko" % settings.rte_sdk], stderr=subprocess.STDOUT, ) if output: output = check_output( ["modprobe", "igb_uio"], stderr=subprocess.STDOUT, ) if output: raise RuntimeError( "Something went wrong with adding 'igb_uio' kernel module: %s" % output.decode("utf-8") ) settings.dpdk_module = module logger.info("loaded DPDK kernel module 'igb_uio'") else: raise RuntimeError("Module '%s' is not supported" % module) settings.dpdk_module = module def remove_dpdk_module(module, settings): if not module: settings.dpdk_module = None return if not is_module_loaded(module): logger.warning("DPDK module not loaded") settings.dpdk_module = None return if module in ["uio_pci_generic", "vfio-pci"]: output = check_output(["modprobe", "-r", module], stderr=subprocess.STDOUT) if output: raise RuntimeError( "Something went wrong with removing '%s' kernel module: %s" % (module, output.decode("utf-8")) ) logger.info("removed kernel module '%s'", module) elif module == "igb_uio": output = check_output(["rmmod", "igb_uio"], stderr=subprocess.STDOUT) if output: raise RuntimeError( "Something went wrong with removing 'igb_uio' kernel module: %s" % output.decode("utf-8") ) logger.info("removed 'igb_uio' kernel module") settings.dpdk_module = None def insert_kni_module(args, settings): kni_module = "rte_kni" if is_module_loaded(kni_module): logger.info("KNI module already loaded") return if not hasattr(settings, "rte_sdk") or not settings.rte_sdk: raise RuntimeError("Cannot find RTE_SDK value in '%s'" % _SETTINGS_FILE) kni_module_path = glob.glob( f"{settings.rte_sdk}/**/{kni_module}.ko", recursive=True ) if not kni_module_path: raise RuntimeError(f"Cannot find KNI kernel module {kni_module}") kmod_params = args.kni_params output = check_output( ["insmod", kni_module_path[0], kmod_params], stderr=subprocess.STDOUT, ) if output: raise RuntimeError( "Something went wrong with adding KNI kernel module: %s" % output.decode("utf-8") ) settings.kni_module = kni_module logger.info("loaded KNI module") def remove_kni_module(settings): kni_module = "rte_kni" if not is_module_loaded(kni_module): logger.warning("KNI module not loaded") settings.kni_module = None return output = check_output(["rmmod", kni_module], stderr=subprocess.STDOUT) if output: raise RuntimeError( "Something went wrong with removing KNI module: %s" % output.decode("utf-8") ) logger.info("removed KNI kernel module") settings.kni_module = None def get_interface_command(interface, up_or_down): if _IP_TOOLS == "ip": return ["ip", "link", "set", "dev", interface, up_or_down] return ["ifconfig", interface, up_or_down] def set_interfaces_down(interfaces, settings): for interface in interfaces: output = check_output( get_interface_command(interface, "down"), stderr=subprocess.STDOUT ) if output: if ( # pylint:disable=using-constant-test # pylint:disable=bad-continuation msg in output.decode("utf-8") for msg in ["No such device", "Cannot find device"] ): raise RuntimeError("Cannot find interface '%s'" % interface) raise RuntimeError( "Cannot shut down interface %s: %s" % (interface, output.decode("utf-8")) ) logger.info("set interface '%s' down", interface) settings.interfaces = json.dumps(interfaces) def set_interfaces_up(interfaces, settings): for interface in interfaces: output = check_output( get_interface_command(interface, "up"), stderr=subprocess.STDOUT ) if output: if "No such device" in output.decode("utf-8"): raise RuntimeError("cannot find interface '%s'" % interface) logger.error( "error restoring interface '%s': %s", interface, output.decode("utf-8") ) else: logger.info("restored interface '%s'", interface) settings.interfaces = None def is_bound_to_driver(dev_id, driver): dev = _DEVICES[dev_id] return has_driver(dev_id) and dev["Driver_str"] == driver def bind_interfaces(dpdk_module, interfaces, interface_infos, settings): load_device_data() for interface in interfaces: try: dev_id = dev_id_from_dev_name(interface) except ValueError: raise RuntimeError( # pylint:disable=raising-format-tuple "cannot find interface '%s'", interface ) # check if the device is already bound to the driver if is_bound_to_driver(dev_id, dpdk_module): logger.info( "interface '%s' is already bound to DPDK driver '%s'", interface, dpdk_module, ) interface_infos.append( {"dev_id": dev_id, "saved_driver": _DEVICES[dev_id]["Module_str"]} ) continue # if device is not bound, bind it saved_driver = bind_one(dev_id, dpdk_module, quiet=False, force=False) if saved_driver: interface_infos.append( { "dev_id": dev_id, "saved_driver": saved_driver, } ) logger.info( "bound interface '%s' ['%s'] to '%s'", interface, dev_id, dpdk_module ) else: raise RuntimeError("Error binding interface '%s'" % interface) settings.interface_infos = json.dumps(interface_infos) def unbind_interfaces(interface_infos, settings): load_device_data() for interface_info in interface_infos: bind_one( interface_info["dev_id"], interface_info["saved_driver"], quiet=True, force=False, ) logger.info( "bound device '%s' back to '%s'", interface_info["dev_id"], interface_info["saved_driver"], ) settings.interface_infos = None def do_restore(dpdk_module, interfaces, interface_infos, settings, kni_module=None): try: restore_huge_pages() unbind_interfaces(interface_infos, settings) time.sleep(1) set_interfaces_up(interfaces, settings) remove_dpdk_module(dpdk_module, settings) if kni_module: remove_kni_module(settings) except PermissionError: logger.error("insufficient privileges. Please run this utility as 'sudo'") raise except Exception as exc: logger.error("error restoring system: %s", exc) raise exc def handle_restore(_args, settings): dpdk_module = getattr(settings, "dpdk_module", None) kni_module = getattr(settings, "kni_module", None) interfaces = json.loads(getattr(settings, "interfaces", "[]")) interface_infos = json.loads(getattr(settings, "interface_infos", "[]")) do_restore( dpdk_module, interfaces, interface_infos, settings, kni_module=kni_module ) logger.info("RESTORE COMPLETE") def handle_setup(args, settings): interface_infos = [] try: verify_rte_sdk(args, settings) except Exception as exc: logger.error(exc) raise try: setup_huge_pages(args.huge_pages) load_dpdk_module(args.dpdk_module, settings) if args.load_kni: insert_kni_module(args, settings) set_interfaces_down(args.interface, settings) bind_interfaces(args.dpdk_module, args.interface, interface_infos, settings) logger.info("SETUP COMPLETE") except PermissionError: logger.error("insufficient privileges. Please run this utility as 'sudo'") raise except Exception as exc: logger.error(exc) logger.error("restoring settings:") do_restore( args.dpdk_module, args.interface, interface_infos, settings, kni_module=args.load_kni, ) raise exc def handle_status(_args, _settings): show_status("all") def parse_args(): parser = argparse.ArgumentParser( description=usage(), formatter_class=argparse.RawTextHelpFormatter ) subparsers = parser.add_subparsers(title="Available options", dest="cmd") subparsers.required = True parser_setup = subparsers.add_parser("setup", help="setup DPDK on your machine") parser_setup.set_defaults(func=handle_setup) parser_setup.add_argument( "-g", "--huge-pages", metavar="AMOUNT", type=int, required=True, help="amount of huge pages to allocate (huge pages are needed for DPDK's memory allocations)", # pylint:disable=line-too-long ) parser_setup.add_argument( "-i", "--interface", metavar="NIC_NAME", type=str, nargs="+", required=True, help="the name of a network interface (e.g eth1) that will be unbound from Linux and move to DPDK control", # pylint:disable=line-too-long ) parser_setup.add_argument( "-m", "--dpdk-module", choices=["igb_uio", "uio_pci_generic", "vfio-pci"], default="igb_uio", help="the DPDK module to install. If not specified the default is 'igb_uio'", ) parser_setup.add_argument( "-k", "--load-kni", action="store_true", help="install the KNI kernel module (not loaded by default)", ) parser_setup.add_argument( "-p", "--kni-params", type=str, default="carrier=on", help="optional parameters for installing the KNI kernel module", ) parser_setup.add_argument( "-v", "--verbose", action="store_true", help="print more verbose output", ) parser_setup.add_argument( "-r", "--rte-sdk", type=str, help="DPDK home directory", ) parser_status = subparsers.add_parser( "status", help="display current Ethernet device settings" ) parser_status.set_defaults(func=handle_status) parser_status.add_argument( "-v", "--verbose", action="store_true", help="print more verbose output", ) parser_restore = subparsers.add_parser( "restore", help="tear down DPDK and restore the system to its original state" ) parser_restore.set_defaults(func=handle_restore) parser_restore.add_argument( "-v", "--verbose", action="store_true", help="print more verbose output", ) return parser.parse_args() def init_logger(args): if args.verbose: logger.setLevel(logging.DEBUG) else: logger.setLevel(logging.INFO) log_format = "[%(levelname)s] %(message)s" formatter = logging.Formatter(log_format) handler = logging.StreamHandler() handler.setFormatter(formatter) logger.addHandler(handler) class Settings: @classmethod def load(cls, settings_file_path): settings = cls() if os.path.exists(settings_file_path): logger.debug("loading from settings file '%s':", settings_file_path) with open(settings_file_path, "r") as settings_file: for line in settings_file.readlines(): line = line.rstrip() if line: logger.debug(" %s", line) [key, value] = line.split("=") setattr(settings, key.lower(), value) return settings def save(self, settings_file_path): with open(settings_file_path, "w") as settings_file: logger.debug("saving to settings file '%s':", settings_file_path) for attr, value in self.__dict__.items(): if value: line = "%s=%s" % (attr.upper(), value) logger.debug(" %s", line) settings_file.write("{line}\n".format(line=line)) def main(): global _IP_TOOLS # pylint:disable=global-statement # check if lspci is installed, suppress any output with open(os.devnull, "w") as devnull: if subprocess.call(["which", "lspci"], stdout=devnull, stderr=devnull) != 0: sys.exit("'lspci' not found - please install 'pciutils'") ip_installed = ( subprocess.call(["which", "ip"], stdout=devnull, stderr=devnull) == 0 ) ifconfig_installed = ( subprocess.call(["which", "ifconfig"], stdout=devnull, stderr=devnull) == 0 ) if ip_installed: _IP_TOOLS = "ip" elif ifconfig_installed: _IP_TOOLS = "ifconfig" else: sys.exit( "both 'ip' and 'ifconfig' are not found - please install one of them" ) load_device_data() args = parse_args() init_logger(args) settings_file_full_path = os.path.join( os.path.dirname(os.path.realpath(__file__)), _SETTINGS_FILE, ) settings = Settings.load(settings_file_full_path) try: args.func(args, settings) settings.save(settings_file_full_path) except: # pylint:disable=bare-except pass if __name__ == "__main__": main()