vmclock: test snapshot safety features

Extend VMClock integration tests to also account for the
vm_generation_counter field and notification support flag.

Signed-off-by: Babis Chalios <bchalios@amazon.es>

Author: bchalios

tests/conftest.py

@@ -236,6 +236,13 @@ def bin_vsock_path(test_fc_session_root_path):
     build_tools.gcc_compile("host_tools/vsock_helper.c", vsock_helper_bin_path)
     yield vsock_helper_bin_path
 
+@pytest.fixture(scope="session")
+def bin_vmclock_path(test_fc_session_root_path):
+    """Build a simple util for test VMclock device"""
+    vmclock_helper_bin_path = os.path.join(test_fc_session_root_path, "vmclock")
+    build_tools.gcc_compile("host_tools/vmclock.c", vmclock_helper_bin_path)
+    yield vmclock_helper_bin_path
+
 
 @pytest.fixture(scope="session")
 def bin_vmclock_path(test_fc_session_root_path):

tests/host_tools/vmclock-abi.h

@@ -115,6 +115,17 @@ struct vmclock_abi {
 	 * bit again after the update, using the about-to-be-valid fields.
 	 */
 #define VMCLOCK_FLAG_TIME_MONOTONIC		(1 << 7)
+	/*
+	 * If the VM_GEN_COUNTER_PRESENT flag is set, the hypervisor will
+	 * bump the vm_generation_counter field every time the guest is
+	 * loaded from some save state (restored from a snapshot).
+	 */
+#define VMCLOCK_FLAG_VM_GEN_COUNTER_PRESENT     (1 << 8)
+	/*
+	 * If the NOTIFICATION_PRESENT flag is set, the hypervisor will send
+	 * a notification every time it updates seq_count to a new even number.
+	 */
+#define VMCLOCK_FLAG_NOTIFICATION_PRESENT       (1 << 9)
 
 	__u8 pad[2];
 	__u8 clock_status;
@@ -177,6 +188,19 @@ struct vmclock_abi {
 	__le64 time_frac_sec;		/* Units of 1/2^64 of a second */
 	__le64 time_esterror_nanosec;
 	__le64 time_maxerror_nanosec;
+
+	/*
+	 * This field changes to another non-repeating value when the VM
+	 * is loaded from a snapshot. This event, typically, represents a
+	 * "jump" forward in time. As a result, in this case as well, the
+	 * guest needs to discard any calibrarion against external sources.
+	 * Loading a snapshot in a VM has different semantics than other VM
+	 * events such as live migration, i.e. apart from re-adjusting guest
+	 * clocks a guest user space might want to discard UUIDs, reset
+	 * network connections or reseed entropy, etc. As a result, we
+	 * use a dedicated marker for such events.
+	 */
+	__le64 vm_generation_counter;
 };
 
 #endif /*  __VMCLOCK_ABI_H__ */

tests/host_tools/vmclock.c

@@ -1,12 +1,12 @@
 // Copyright 2025 Amazon.com, Inc. or its affiliates. All Rights Reserved.
 // SPDX-License-Identifier: Apache-2.0
 
-#include <errno.h>
 #include <stdatomic.h>
 #include <stdint.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
+#include <sys/epoll.h>
 #include <sys/stat.h>
 #include <fcntl.h>
 #include <sys/mman.h>
@@ -16,23 +16,26 @@
 
 const char *VMCLOCK_DEV_PATH = "/dev/vmclock0";
 
-int get_vmclock_handle(struct vmclock_abi **vmclock)
+int open_vmclock(void)
 {
   int fd = open(VMCLOCK_DEV_PATH, 0);
-  if (fd == -1)
-    goto out_err;
+  if (fd == -1) {
+    perror("open");
+    exit(1);
+  }
 
-  void *ptr = mmap(NULL, sizeof(struct vmclock_abi), PROT_READ, MAP_SHARED, fd, 0);
-  if (ptr == MAP_FAILED)
-    goto out_err_mmap;
+  return fd;
+}
 
-  *vmclock = ptr;
-  return 0;
+struct vmclock_abi *get_vmclock_handle(int fd)
+{
+  void *ptr = mmap(NULL, sizeof(struct vmclock_abi), PROT_READ, MAP_SHARED, fd, 0);
+  if (ptr == MAP_FAILED) {
+    perror("mmap");
+    exit(1);
+  }
 
-out_err_mmap:
-  close(fd);
-out_err:
-  return errno;
+  return ptr;
 }
 
 #define READ_VMCLOCK_FIELD_FN(type, field)                 \
@@ -56,23 +59,136 @@ type read##_##field (struct vmclock_abi *vmclock) {        \
 }
 
 READ_VMCLOCK_FIELD_FN(uint64_t, disruption_marker);
+READ_VMCLOCK_FIELD_FN(uint64_t, vm_generation_counter);
 
-int main()
+/*
+ * Read `vmclock_abi` structure using a file descriptor pointing to
+ * `/dev/vmclock0`.
+ */
+void read_vmclock(int fd, struct vmclock_abi *vmclock)
 {
-  struct vmclock_abi *vmclock;
+  int ret;
 
-  int err = get_vmclock_handle(&vmclock);
-  if (err) {
-    printf("Could not mmap vmclock struct: %s\n", strerror(err));
+  /*
+   * Use `pread()`, since the device doesn't implement lseek(), so
+   * we can't reset `fp`.
+   */
+  ret = pread(fd, vmclock, sizeof(*vmclock), 0);
+  if (ret < 0) {
+    perror("read");
+    exit(1);
+  } else if (ret < (int) sizeof(*vmclock)) {
+    fprintf(stderr, "We don't handle partial writes (%d). Exiting!\n", ret);
     exit(1);
   }
+}
+
+void print_vmclock(struct vmclock_abi *vmclock)
+{
+  if (vmclock->flags & VMCLOCK_FLAG_VM_GEN_COUNTER_PRESENT) {
+    printf("VMCLOCK_FLAG_VM_GEN_COUNTER_PRESENT: true\n");
+  } else {
+    printf("VMCLOCK_FLAG_VM_GEN_COUNTER_PRESENT: false\n");
+  }
+
+  if (vmclock->flags & VMCLOCK_FLAG_NOTIFICATION_PRESENT) {
+    printf("VMCLOCK_FLAG_NOTIFICATION_PRESENT: true\n");
+  } else {
+    printf("VMCLOCK_FLAG_NOTIFICATION_PRESENT: false\n");
+  }
 
   printf("VMCLOCK_MAGIC: 0x%x\n", vmclock->magic);
   printf("VMCLOCK_SIZE: 0x%x\n", vmclock->size);
   printf("VMCLOCK_VERSION: %u\n", vmclock->version);
   printf("VMCLOCK_CLOCK_STATUS: %u\n", vmclock->clock_status);
   printf("VMCLOCK_COUNTER_ID: %u\n", vmclock->counter_id);
   printf("VMCLOCK_DISRUPTION_MARKER: %lu\n", read_disruption_marker(vmclock));
+  printf("VMCLOCK_VM_GENERATION_COUNTER: %lu\n", read_vm_generation_counter(vmclock));
+  fflush(stdout);
+}
+
+void run_poll(int fd) 
+{
+  struct vmclock_abi vmclock;
+  int epfd, ret, nfds;
+  struct epoll_event ev;
+
+  read_vmclock(fd, &vmclock);
+  print_vmclock(&vmclock);
+
+  epfd = epoll_create(1);
+  if (epfd < 0) {
+    perror("epoll_create");
+    exit(1);
+  }
+
+  ev.events = EPOLLIN | EPOLLRDNORM;
+  ev.data.fd = fd;
+  ret = epoll_ctl(epfd, EPOLL_CTL_ADD, fd, &ev);
+  if (ret < 0) {
+    perror("epoll_add");
+    exit(1);
+  }
+  
+  while (1) {
+    nfds = epoll_wait(epfd, &ev, 1, -1);
+    if (nfds < 0) {
+      perror("epoll_wait");
+      exit(1);
+    }
+
+    if (ev.data.fd != fd) {
+      fprintf(stderr, "Unknown file descriptor %d\n", ev.data.fd);
+      exit(1);
+    }
+
+    if (ev.events & EPOLLHUP) {
+      fprintf(stderr, "Device does not support notifications. Stop polling\n");
+      exit(1);
+    } else if (ev.events & EPOLLIN) {
+      fprintf(stdout, "Got VMClock notification\n");
+      read_vmclock(fd, &vmclock);
+      print_vmclock(&vmclock);
+    }
+  }
+}
+
+void print_help_message()
+{
+    fprintf(stderr, "usage: vmclock MODE\n");
+    fprintf(stderr, "Available modes:\n");
+    fprintf(stderr, "   -r\tRead vmclock_abi using read()\n");
+    fprintf(stderr, "   -m\tRead vmclock_abi using mmap()\n");
+    fprintf(stderr, "   -p\tPoll VMClock for changes\n");
+}
+
+int main(int argc, char *argv[])
+{
+  int fd;
+  struct vmclock_abi vmclock, *vmclock_ptr;
+
+  if (argc != 2) {
+    print_help_message();
+    exit(1);
+  }
+
+  fd = open_vmclock();
+
+  if (!strncmp(argv[1], "-r", 3)) {
+    printf("Reading VMClock with read()\n");
+    read_vmclock(fd, &vmclock);
+    print_vmclock(&vmclock);
+  } else if (!strncmp(argv[1], "-m", 3)) {
+    printf("Reading VMClock with mmap()\n");
+    vmclock_ptr = get_vmclock_handle(fd);
+    print_vmclock(vmclock_ptr);
+  } else if (!strncmp(argv[1], "-p", 3)) {
+    printf("Polling VMClock\n");
+    run_poll(fd);
+  } else {
+    print_help_message();
+    exit(1);
+  }
 
   return 0;
 }

tests/integration_tests/functional/test_vmclock.py

@@ -21,46 +21,107 @@ def vm_with_vmclock(uvm_plain_acpi, bin_vmclock_path):
     yield basevm
 
 
-def parse_vmclock(vm):
+def parse_vmclock(vm, use_mmap=False):
     """Parse the VMclock struct inside the guest and return a dictionary with its fields"""
-    _, stdout, _ = vm.ssh.check_output("/tmp/vmclock")
+
+    cmd = "/tmp/vmclock -m" if use_mmap else "/tmp/vmclock -r"
+    _, stdout, _ = vm.ssh.check_output(cmd)
+    fields = stdout.strip().split("\n")
+    if use_mmap:
+        assert fields[0] == "Reading VMClock with mmap()"
+    else:
+        assert fields[0] == "Reading VMClock with read()"
+
+    return dict(item.split(": ") for item in fields if item.startswith("VMCLOCK"))
+
+
+def parse_vmclock_from_poll(vm, expected_notifications):
+    """Parse the output of the 'vmclock -p' command in the guest"""
+
+    _, stdout, _ = vm.ssh.check_output("cat /tmp/vmclock.out")
     fields = stdout.strip().split("\n")
-    return dict(item.split(": ") for item in fields)
+
+    nr_notifications = 0
+    for line in fields:
+        if line == "Got VMClock notification":
+            nr_notifications += 1
+
+    # assert nr_notifications == expected_notifications
+    return dict(item.split(": ") for item in fields if item.startswith("VMCLOCK"))
 
 
 @pytest.mark.skipif(
     platform.machine() != "x86_64",
     reason="VMClock device is currently supported only on x86 systems",
 )
-def test_vmclock_fields(vm_with_vmclock):
+@pytest.mark.parametrize("use_mmap", [False, True], ids=["read()", "mmap()"])
+def test_vmclock_read_fields(vm_with_vmclock, use_mmap):
     """Make sure that we expose the expected values in the VMclock struct"""
     vm = vm_with_vmclock
-    vmclock = parse_vmclock(vm)
+    vmclock = parse_vmclock(vm, use_mmap)
 
+    assert vmclock["VMCLOCK_FLAG_VM_GEN_COUNTER_PRESENT"] == "true"
+    assert vmclock["VMCLOCK_FLAG_NOTIFICATION_PRESENT"] == "true"
     assert vmclock["VMCLOCK_MAGIC"] == "0x4b4c4356"
     assert vmclock["VMCLOCK_SIZE"] == "0x1000"
     assert vmclock["VMCLOCK_VERSION"] == "1"
     assert vmclock["VMCLOCK_CLOCK_STATUS"] == "0"
     assert vmclock["VMCLOCK_COUNTER_ID"] == "255"
     assert vmclock["VMCLOCK_DISRUPTION_MARKER"] == "0"
+    assert vmclock["VMCLOCK_VM_GENERATION_COUNTER"] == "0"
 
 
 @pytest.mark.skipif(
     platform.machine() != "x86_64",
     reason="VMClock device is currently supported only on x86 systems",
 )
-def test_snapshot_update(vm_with_vmclock, microvm_factory, snapshot_type):
-    """Test that `disruption_marker` is updated upon snapshot resume"""
+@pytest.mark.parametrize("use_mmap", [False, True], ids=["read()", "mmap()"])
+def test_snapshot_update(vm_with_vmclock, microvm_factory, snapshot_type, use_mmap):
+    """Test that `disruption_marker` and `vm_generation_counter` are updated
+    upon snapshot resume"""
     basevm = vm_with_vmclock
 
-    vmclock = parse_vmclock(basevm)
+    vmclock = parse_vmclock(basevm, use_mmap)
+    assert vmclock["VMCLOCK_FLAG_VM_GEN_COUNTER_PRESENT"] == "true"
+    assert vmclock["VMCLOCK_FLAG_NOTIFICATION_PRESENT"] == "true"
+    assert vmclock["VMCLOCK_DISRUPTION_MARKER"] == "0"
+    assert vmclock["VMCLOCK_VM_GENERATION_COUNTER"] == "0"
+
+    snapshot = basevm.make_snapshot(snapshot_type)
+    basevm.kill()
+
+    for i, vm in enumerate(
+        microvm_factory.build_n_from_snapshot(snapshot, 5, incremental=True)
+    ):
+        vmclock = parse_vmclock(vm, use_mmap)
+        assert vmclock["VMCLOCK_DISRUPTION_MARKER"] == f"{i+1}"
+        assert vmclock["VMCLOCK_VM_GENERATION_COUNTER"] == f"{i+1}"
+
+
+# TODO: remove this skip when we backport VMClock snapshot safety patches to 5.10 and 6.1
+@pytest.mark.skip(
+    reason="Skip until we get guest microVM kernels with support for the notification mechanism",
+)
+def test_vmclock_notifications(vm_with_vmclock, microvm_factory, snapshot_type):
+    """Test that Firecracker will send a notification on snapshot load"""
+    basevm = vm_with_vmclock
+
+    # Launch vmclock utility in polling mode
+    basevm.ssh.check_output("/tmp/vmclock -p > /tmp/vmclock.out 2>&1 &")
+
+    # We should not have received any notification yet
+    vmclock = parse_vmclock_from_poll(basevm, 0)
+    assert vmclock["VMCLOCK_FLAG_VM_GEN_COUNTER_PRESENT"] == "true"
+    assert vmclock["VMCLOCK_FLAG_NOTIFICATION_PRESENT"] == "true"
     assert vmclock["VMCLOCK_DISRUPTION_MARKER"] == "0"
+    assert vmclock["VMCLOCK_VM_GENERATION_COUNTER"] == "0"
 
     snapshot = basevm.make_snapshot(snapshot_type)
     basevm.kill()
 
     for i, vm in enumerate(
         microvm_factory.build_n_from_snapshot(snapshot, 5, incremental=True)
     ):
-        vmclock = parse_vmclock(vm)
+        vmclock = parse_vmclock_from_poll(vm, i + 1)
         assert vmclock["VMCLOCK_DISRUPTION_MARKER"] == f"{i+1}"
+        assert vmclock["VMCLOCK_VM_GENERATION_COUNTER"] == f"{i+1}"