From 5c1f85cc1794d38d2a12258e9b68e280da887cbe Mon Sep 17 00:00:00 2001 From: Stefan Kober Date: Fri, 7 Mar 2025 14:21:27 +0100 Subject: [PATCH 1/6] libvirt/ch: add Cloud Hypervisor support On-behalf-of: SAP philipp.schuster@sap.com On-behalf-of: SAP stefan.kober@sap.com Co-authored-by: Stefan Kober Co-authored-by: Philipp Schuster --- nova/conf/libvirt.py | 2 +- nova/privsep/libvirt.py | 17 +++++++++++------ nova/virt/libvirt/blockinfo.py | 3 +++ nova/virt/libvirt/driver.py | 20 +++++++++++++++++++- nova/virt/libvirt/vif.py | 5 ++++- 5 files changed, 38 insertions(+), 9 deletions(-) diff --git a/nova/conf/libvirt.py b/nova/conf/libvirt.py index 9d59beed833..cec163100d1 100644 --- a/nova/conf/libvirt.py +++ b/nova/conf/libvirt.py @@ -104,7 +104,7 @@ """), cfg.StrOpt('virt_type', default='kvm', - choices=('kvm', 'lxc', 'qemu', 'parallels'), + choices=('kvm', 'lxc', 'qemu', 'parallels', 'ch'), help=""" Describes the virtualization type (or so called domain type) libvirt should use. diff --git a/nova/privsep/libvirt.py b/nova/privsep/libvirt.py index 6ca99b98b42..4c0d9d877d4 100644 --- a/nova/privsep/libvirt.py +++ b/nova/privsep/libvirt.py @@ -156,15 +156,20 @@ def readpty(path): # exception here... Some platforms (I'm looking at you Windows) # don't have a fcntl and we may as well let them know that # with an ImportError, not that they should be calling this at all. - import fcntl + import select try: + epoll = select.epoll() with open(path, 'r') as f: - current_flags = fcntl.fcntl(f.fileno(), fcntl.F_GETFL) - fcntl.fcntl(f.fileno(), fcntl.F_SETFL, - current_flags | os.O_NONBLOCK) - - return f.read() + os.set_blocking(f.fileno(), False) + epoll.register(f.fileno(), select.EPOLLIN) + poll_list = epoll.poll(1) + data = '' + for _ in poll_list: + data += f.read() + epoll.unregister(f.fileno()) + epoll.close() + return data except Exception as exc: # NOTE(mikal): dear internet, I see you looking at me with your diff --git a/nova/virt/libvirt/blockinfo.py b/nova/virt/libvirt/blockinfo.py index 4efc6fbaeb1..74bd23f00ed 100644 --- a/nova/virt/libvirt/blockinfo.py +++ b/nova/virt/libvirt/blockinfo.py @@ -95,6 +95,7 @@ 'qemu': ['virtio', 'scsi', 'ide', 'usb', 'fdc', 'sata'], 'kvm': ['virtio', 'scsi', 'ide', 'usb', 'fdc', 'sata'], 'lxc': ['lxc'], + 'ch': [ 'virtio' ], 'parallels': ['ide', 'scsi'], # we no longer support UML or Xen, but we keep track of their bus types so # we can reject them for other virt types @@ -276,6 +277,8 @@ def get_disk_bus_for_device_type(instance, return "ide" elif device_type == "disk": return "scsi" + elif virt_type == "ch": + return "virtio" else: # If virt-type not in list then it is unsupported raise exception.UnsupportedVirtType(virt=virt_type) diff --git a/nova/virt/libvirt/driver.py b/nova/virt/libvirt/driver.py index 71dca0410ac..0c030708be6 100644 --- a/nova/virt/libvirt/driver.py +++ b/nova/virt/libvirt/driver.py @@ -1379,6 +1379,8 @@ def _uri(): uri = CONF.libvirt.connection_uri or 'lxc:///' elif CONF.libvirt.virt_type == 'parallels': uri = CONF.libvirt.connection_uri or 'parallels:///system' + elif CONF.libvirt.virt_type == 'ch': + uri = CONF.libvirt.connection_uri or 'ch:///session' else: uri = CONF.libvirt.connection_uri or 'qemu:///system' return uri @@ -4494,7 +4496,8 @@ def get_console_output(self, context, instance): path_sources = [ ('file', "./devices/console[@type='file']/source[@path]", 'path'), ('tcp', "./devices/console[@type='tcp']/log[@file]", 'file'), - ('pty', "./devices/console[@type='pty']/source[@path]", 'path')] + ('pty', "./devices/console[@type='pty']/source[@path]", 'path'), + ('pty', "./devices/serial[@type='pty']/source[@path]", 'path'),] console_type = "" console_path = "" for c_type, epath, attrib in path_sources: @@ -6735,6 +6738,9 @@ def _configure_guest_by_virt_type( guest.os_init_path = "/sbin/init" guest.os_cmdline = CONSOLE guest.os_init_env["product_name"] = "OpenStack Nova" + elif CONF.libvirt.virt_type == "ch": + guest.virt_type = 'kvm' + guest.os_kernel = "/var/lib/nova/hypervisor-fw" elif CONF.libvirt.virt_type == "parallels": if guest.os_type == fields.VMMode.EXE: guest.os_init_path = "/sbin/init" @@ -6780,6 +6786,11 @@ def _create_consoles(self, guest_cfg, instance, flavor, image_meta): self._create_pty_device( guest_cfg, vconfig.LibvirtConfigGuestConsole, log_path=log_path) + elif CONF.libvirt.virt_type == "ch": + consolepty = vconfig.LibvirtConfigGuestSerial() + consolepty.type = "pty" + guest_cfg.add_device(consolepty) + else: # qemu, kvm if self._is_s390x_guest(image_meta): self._create_consoles_s390x( @@ -6969,6 +6980,8 @@ def _guest_add_usb_root_controller(self, guest, image_meta): here explicitly so that we can _disable_ it (by setting the model to 'none') if it's not necessary. """ + if CONF.libvirt.virt_type == "ch": + return usbhost = vconfig.LibvirtConfigGuestUSBHostController() usbhost.index = 0 # an unset model means autodetect, while 'none' means don't add a @@ -7142,6 +7155,9 @@ def _get_guest_config(self, instance, network_info, image_meta, self._add_rng_device(guest, flavor, image_meta) self._add_vtpm_device(guest, flavor, instance, image_meta) + if CONF.libvirt.virt_type == "ch": + self._add_rng_device(guest, flavor, image_meta) + if self._guest_needs_pcie(guest): self._guest_add_pcie_root_ports(guest) @@ -7378,6 +7394,8 @@ def _guest_add_accel_pci_devices(self, guest, accel_info): def _guest_add_video_device(guest): if CONF.libvirt.virt_type == 'lxc': return False + elif CONF.libvirt.virt_type == "ch": + return False # NB some versions of libvirt support both SPICE and VNC # at the same time. We're not trying to second guess which diff --git a/nova/virt/libvirt/vif.py b/nova/virt/libvirt/vif.py index 6e9069fa50f..30456e678d2 100644 --- a/nova/virt/libvirt/vif.py +++ b/nova/virt/libvirt/vif.py @@ -78,6 +78,9 @@ network_model.VIF_MODEL_RTL8139, network_model.VIF_MODEL_E1000, ], + 'ch': [ + network_model.VIF_MODEL_VIRTIO, + ], } @@ -172,7 +175,7 @@ def get_vif_model(self, image_meta=None, vif_model=None): # If the virt type is KVM/QEMU/VZ(Parallels), then use virtio according # to the global config parameter if (model is None and CONF.libvirt.virt_type in - ('kvm', 'qemu', 'parallels') and + ('kvm', 'qemu', 'parallels', 'ch') and CONF.libvirt.use_virtio_for_bridges): model = network_model.VIF_MODEL_VIRTIO From d5e2da0b927d8c21f4de1dbdb9844d488a6ea0b6 Mon Sep 17 00:00:00 2001 From: Philipp Schuster Date: Mon, 30 Jun 2025 18:25:51 +0200 Subject: [PATCH 2/6] libvirt/ch: xxx fix rng device In the old patchset from March, things used to work like this. rng device probably not needed by SAP at the moment. --- nova/virt/libvirt/driver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nova/virt/libvirt/driver.py b/nova/virt/libvirt/driver.py index 0c030708be6..c2d0f8a2536 100644 --- a/nova/virt/libvirt/driver.py +++ b/nova/virt/libvirt/driver.py @@ -7156,7 +7156,7 @@ def _get_guest_config(self, instance, network_info, image_meta, self._add_vtpm_device(guest, flavor, instance, image_meta) if CONF.libvirt.virt_type == "ch": - self._add_rng_device(guest, flavor, image_meta) + return if self._guest_needs_pcie(guest): self._guest_add_pcie_root_ports(guest) From 82cfd59d26ef1522750f9b36b138392cf81e91b9 Mon Sep 17 00:00:00 2001 From: Philipp Schuster Date: Mon, 7 Jul 2025 10:57:16 +0200 Subject: [PATCH 3/6] libvirt/ch: apply SAP patches xxx Patchset from christian --- nova/virt/libvirt/blockinfo.py | 2 +- nova/virt/libvirt/driver.py | 10 ++++------ nova/virt/libvirt/host.py | 1 + 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/nova/virt/libvirt/blockinfo.py b/nova/virt/libvirt/blockinfo.py index 74bd23f00ed..124aa1b7e5c 100644 --- a/nova/virt/libvirt/blockinfo.py +++ b/nova/virt/libvirt/blockinfo.py @@ -95,7 +95,7 @@ 'qemu': ['virtio', 'scsi', 'ide', 'usb', 'fdc', 'sata'], 'kvm': ['virtio', 'scsi', 'ide', 'usb', 'fdc', 'sata'], 'lxc': ['lxc'], - 'ch': [ 'virtio' ], + 'ch': ['virtio'], 'parallels': ['ide', 'scsi'], # we no longer support UML or Xen, but we keep track of their bus types so # we can reject them for other virt types diff --git a/nova/virt/libvirt/driver.py b/nova/virt/libvirt/driver.py index c2d0f8a2536..f5aca2aaa0b 100644 --- a/nova/virt/libvirt/driver.py +++ b/nova/virt/libvirt/driver.py @@ -1380,7 +1380,7 @@ def _uri(): elif CONF.libvirt.virt_type == 'parallels': uri = CONF.libvirt.connection_uri or 'parallels:///system' elif CONF.libvirt.virt_type == 'ch': - uri = CONF.libvirt.connection_uri or 'ch:///session' + uri = CONF.libvirt.connection_uri or 'ch:///system' else: uri = CONF.libvirt.connection_uri or 'qemu:///system' return uri @@ -1391,6 +1391,7 @@ def _live_migration_uri(dest): 'kvm': 'qemu+%(scheme)s://%(dest)s/system', 'qemu': 'qemu+%(scheme)s://%(dest)s/system', 'parallels': 'parallels+tcp://%(dest)s/system', + 'ch': 'ch+%(scheme)s://%(dest)s/system', } dest = oslo_netutils.escape_ipv6(dest) @@ -4497,7 +4498,7 @@ def get_console_output(self, context, instance): ('file', "./devices/console[@type='file']/source[@path]", 'path'), ('tcp', "./devices/console[@type='tcp']/log[@file]", 'file'), ('pty', "./devices/console[@type='pty']/source[@path]", 'path'), - ('pty', "./devices/serial[@type='pty']/source[@path]", 'path'),] + ('pty', "./devices/serial[@type='pty']/source[@path]", 'path'), ] console_type = "" console_path = "" for c_type, epath, attrib in path_sources: @@ -6740,7 +6741,7 @@ def _configure_guest_by_virt_type( guest.os_init_env["product_name"] = "OpenStack Nova" elif CONF.libvirt.virt_type == "ch": guest.virt_type = 'kvm' - guest.os_kernel = "/var/lib/nova/hypervisor-fw" + guest.os_kernel = "/usr/share/cloud-hypervisor/CLOUDHV_EFI.fd" elif CONF.libvirt.virt_type == "parallels": if guest.os_type == fields.VMMode.EXE: guest.os_init_path = "/sbin/init" @@ -7155,9 +7156,6 @@ def _get_guest_config(self, instance, network_info, image_meta, self._add_rng_device(guest, flavor, image_meta) self._add_vtpm_device(guest, flavor, instance, image_meta) - if CONF.libvirt.virt_type == "ch": - return - if self._guest_needs_pcie(guest): self._guest_add_pcie_root_ports(guest) diff --git a/nova/virt/libvirt/host.py b/nova/virt/libvirt/host.py index f67ccf9bbf7..2ae47c2a702 100644 --- a/nova/virt/libvirt/host.py +++ b/nova/virt/libvirt/host.py @@ -1241,6 +1241,7 @@ def write_instance_config(self, xml): :returns: an instance of Guest """ + LOG.info(xml) domain = self.get_connection().defineXML(xml) return libvirt_guest.Guest(domain) From ded52a6265961365843e44cbdd43bd11ed4a7f07 Mon Sep 17 00:00:00 2001 From: Stefan Kober Date: Tue, 15 Jul 2025 12:59:12 +0200 Subject: [PATCH 4/6] libvirt: fix readpty Cloud Hypervisor uses a pty as a channel to and from the guest. To allow openstack to display the console output of the VM, the pty must be read. Reading from the pty has proven tricky, as it seems some input needs to be provided and order to get output again. Therefore, we send some null byte to the pty prior to reading. The reading is also done in a loop to receive all available output. On-behalf-of: SAP stefan.kober@sap.com --- nova/privsep/libvirt.py | 70 ++++++++++++++++++++++++++++++----------- 1 file changed, 51 insertions(+), 19 deletions(-) diff --git a/nova/privsep/libvirt.py b/nova/privsep/libvirt.py index 4c0d9d877d4..08618eab99a 100644 --- a/nova/privsep/libvirt.py +++ b/nova/privsep/libvirt.py @@ -146,30 +146,62 @@ def unplug_plumgrid_vif(dev): processutils.execute('ifc_ctl', 'gateway', 'ifdown', dev) processutils.execute('ifc_ctl', 'gateway', 'del_port', dev) +def readpty_once(path): + import select + + data = bytearray() + epoll = select.epoll() + + with open(path, "w") as f: + # Writing a null seems to trigger new output while not being + # recognized as a newline or similar by the sender. + f.write("\0") + f.flush() + + with open(path, "rb") as f: + os.set_blocking(f.fileno(), False) + epoll.register(f.fileno(), select.EPOLLIN) + poll_list = epoll.poll(0.1) + for _ in poll_list: + data += f.read() + epoll.unregister(f.fileno()) + epoll.close() + + return data.decode("utf-8", errors="ignore") + @nova.privsep.sys_admin_pctxt.entrypoint def readpty(path): - # TODO(mikal): I'm not a huge fan that we don't enforce a valid pty path - # here, but I haven't come up with a great way of doing that. + """ + The pty created by Cloud Hypervisor is a bit tricky. It seems to require + some kind of input to deliver output again. Therefore, we send in a '\0' + byte before reading (the null byte does not seem to trigger any + interactions with the pty like a newline would do). + The reading is done in a loop and we cancel reading when no data is + returned anymore. Doing so has proven to deliver all the output available + in the pty. + """ + # We track how many times we read zero bytes form the pty + consecutive_zero = 0 + data = "" + try: + # We finish reading once we read zero multiple times continuously. The range + # is just a safeguard to finish reading if the terminal happens to have new + # data all the time. + for _ in range(1000): + out = readpty_once(path) - # NOTE(mikal): I am deliberately not catching the ImportError - # exception here... Some platforms (I'm looking at you Windows) - # don't have a fcntl and we may as well let them know that - # with an ImportError, not that they should be calling this at all. - import select + if len(out) == 0: + consecutive_zero += 1 + else: + consecutive_zero = 0 - try: - epoll = select.epoll() - with open(path, 'r') as f: - os.set_blocking(f.fileno(), False) - epoll.register(f.fileno(), select.EPOLLIN) - poll_list = epoll.poll(1) - data = '' - for _ in poll_list: - data += f.read() - epoll.unregister(f.fileno()) - epoll.close() - return data + data += out + + if consecutive_zero > 10: + return data + + return data except Exception as exc: # NOTE(mikal): dear internet, I see you looking at me with your From bf3aec5ad2643b14147562e4d7d9026b5de2c458 Mon Sep 17 00:00:00 2001 From: Stefan Kober Date: Wed, 30 Jul 2025 13:35:02 +0200 Subject: [PATCH 5/6] libvirt/ch: allow NUMA tunings with CHV Currently, the Nova Libvirt driver only enables numa tuning if QEMU/KVM is the hypervisor type. As we have built NUMA support for the Libvirt CHV driver, we now want to enable it in Nova too. --- nova/virt/libvirt/driver.py | 3 ++- nova/virt/libvirt/host.py | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/nova/virt/libvirt/driver.py b/nova/virt/libvirt/driver.py index f5aca2aaa0b..4ae8827ff28 100644 --- a/nova/virt/libvirt/driver.py +++ b/nova/virt/libvirt/driver.py @@ -8694,7 +8694,8 @@ def _has_numa_support(self): if (caps.host.cpu.arch in (fields.Architecture.I686, fields.Architecture.X86_64, fields.Architecture.AARCH64) and - self._host.has_min_version(hv_type=host.HV_DRIVER_QEMU)): + (self._host.has_min_version(hv_type=host.HV_DRIVER_QEMU) or + self._host.has_min_version(hv_type=host.HV_DRIVER_CH))): return True elif (caps.host.cpu.arch in (fields.Architecture.PPC64, fields.Architecture.PPC64LE)): diff --git a/nova/virt/libvirt/host.py b/nova/virt/libvirt/host.py index 2ae47c2a702..3599b6ff78d 100644 --- a/nova/virt/libvirt/host.py +++ b/nova/virt/libvirt/host.py @@ -85,6 +85,7 @@ # This list is for libvirt hypervisor drivers that need special handling. # This is *not* the complete list of supported hypervisor drivers. HV_DRIVER_QEMU = "QEMU" +HV_DRIVER_CH = "CH" SEV_KERNEL_PARAM_FILE = '/sys/module/kvm_amd/parameters/sev' From 485d452caca441949df438ac9811af9846cbc415 Mon Sep 17 00:00:00 2001 From: Stefan Kober Date: Thu, 31 Jul 2025 15:05:11 +0200 Subject: [PATCH 6/6] ch: set host-passthrough as cpu mode per default Nova requires some CPU model like host passthrough, host model, or custom. Otherwise, things like NUMA tunings will not be configured correctly later on. Therefore, we use host-passthrough per default for cloud-hypervisor, which is the only mode currently supported. --- nova/virt/libvirt/driver.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/nova/virt/libvirt/driver.py b/nova/virt/libvirt/driver.py index 4ae8827ff28..43d26c9a21e 100644 --- a/nova/virt/libvirt/driver.py +++ b/nova/virt/libvirt/driver.py @@ -1025,7 +1025,7 @@ def _check_cpu_compatibility(self): mode = CONF.libvirt.cpu_mode models = CONF.libvirt.cpu_models - if (CONF.libvirt.virt_type not in ("kvm", "qemu") and + if (CONF.libvirt.virt_type not in ("kvm", "qemu", "ch") and mode not in (None, 'none')): msg = _("Config requested an explicit CPU model, but " "the current libvirt hypervisor '%s' does not " @@ -5409,6 +5409,10 @@ def _get_guest_cpu_model_config(self, flavor=None, arch=None): if not models: models = ['max'] + elif CONF.libvirt.virt_type == "ch": + # Currently, we only support host-passthrough with Cloud Hypervisor. This + # changes as soon as we support proper CPU profiles. + mode = "host-passthrough" else: if mode is None or mode == "none": return None