diff --git a/core_common.c b/core_common.c index c4b1251..19177d2 100644 --- a/core_common.c +++ b/core_common.c @@ -10,17 +10,6 @@ #define trace_mx_dma_xfer_complete_orphan(xfer_id, status, result) do { } while (0) #endif -bool liveness_enable; -module_param(liveness_enable, bool, 0644); -MODULE_PARM_DESC(liveness_enable, "Enable device liveness ping watchdog (requires ping-capable FW)"); -unsigned int liveness_stall_ms = 1000; -module_param(liveness_stall_ms, uint, 0644); -unsigned int liveness_dead_ms = 5000; -module_param(liveness_dead_ms, uint, 0644); -unsigned int liveness_max_mult = 10; -module_param(liveness_max_mult, uint, 0644); -MODULE_PARM_DESC(liveness_max_mult, "Absolute wait ceiling = timeout_ms * this (clamped to 1..1000) while transport stays alive"); - /******************************************************************************/ /* Descriptor list utilities */ /******************************************************************************/ @@ -266,12 +255,13 @@ void mx_stop_queue_threads(struct mx_pci_dev *mx_pdev) */ static void mx_liveness_watchdog(struct mx_queue *q) { + struct mx_pci_dev *mx_pdev = q->mx_pdev; unsigned long now = jiffies; int outstanding = atomic_read(&q->wait_count) - atomic_read(&q->zombie_wait_count); /* Snapshot + sanitize sysfs-writable params: keep 1 <= stall < dead so a probe * is always attempted before the no-completion DEAD verdict fires. */ - unsigned int dead_ms = max(liveness_dead_ms, 2u); - unsigned int stall_ms = clamp(liveness_stall_ms, 1u, dead_ms - 1); + unsigned int dead_ms = max(READ_ONCE(mx_pdev->liveness_dead_ms), 2u); + unsigned int stall_ms = clamp(READ_ONCE(mx_pdev->liveness_stall_ms), 1u, dead_ms - 1); unsigned long stalled_ms; if (outstanding <= 0) @@ -313,6 +303,7 @@ int mx_submit_handler(void *arg) unsigned long flags; unsigned int idle_count = 0; bool pushed_any; + bool lv_on; while (!kthread_should_stop()) { __swait_event_interruptible_timeout(q->sq_wait, @@ -320,10 +311,11 @@ int mx_submit_handler(void *arg) POLLING_INTERVAL_MSEC); pushed_any = false; + lv_on = READ_ONCE(q->mx_pdev->liveness_enable); spin_lock_irqsave(&q->sq_lock, flags); list_for_each_entry_safe(transfer, tmp, &q->sq_list, entry) { /* Ping outstanding: hold submits until the pong resolves — the liveness probe has priority. */ - if (liveness_enable && atomic_read(&q->lv_inflight)) + if (lv_on && atomic_read(&q->lv_inflight)) break; if (!ops->is_pushable(q)) break; @@ -348,7 +340,7 @@ int mx_submit_handler(void *arg) swake_up_one(&q->cq_wait); } } - if (liveness_enable) + if (lv_on) mx_liveness_watchdog(q); spin_unlock_irqrestore(&q->sq_lock, flags); diff --git a/core_v1.c b/core_v1.c index 6d09626..9d6fa97 100644 --- a/core_v1.c +++ b/core_v1.c @@ -340,6 +340,7 @@ static int init_mx_queue(struct mx_pci_dev* mx_pdev) mx_mbox_init(&queue->cq_mbox, (uint64_t)ctx_addr, (uint64_t)data_addr, ctx); queue->common.dev = dev; + queue->common.mx_pdev = mx_pdev; queue->common.ops = &v1_queue_ops; spin_lock_init(&queue->common.sq_lock); INIT_LIST_HEAD(&queue->common.sq_list); diff --git a/core_v2.c b/core_v2.c index cc55a94..8f088fe 100644 --- a/core_v2.c +++ b/core_v2.c @@ -358,6 +358,7 @@ static void configure_queue(struct mx_pci_dev *mx_pdev, struct mx_queue_v2 *queu uint64_t __iomem *dbs = mx_pdev->bar + NVME_REG_DBS; queue->common.dev = &mx_pdev->pdev->dev; + queue->common.mx_pdev = mx_pdev; queue->qid = qid; queue->sq_tail = 0; queue->sq_head = 0; diff --git a/init.c b/init.c index 16f2b0e..03d34aa 100644 --- a/init.c +++ b/init.c @@ -161,6 +161,148 @@ static int set_dma_addressing(struct pci_dev *pdev) return 0; } +static ssize_t liveness_enable_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct mx_pci_dev *mx_pdev = dev_get_drvdata(dev); + + if (!mx_pdev) + return -ENODEV; + return sysfs_emit(buf, "%d\n", READ_ONCE(mx_pdev->liveness_enable)); +} + +static ssize_t liveness_enable_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count) +{ + struct mx_pci_dev *mx_pdev = dev_get_drvdata(dev); + struct mx_queue *q; + unsigned long flags; + bool val; + int ret; + + if (!mx_pdev) + return -ENODEV; + + ret = kstrtobool(buf, &val); + if (ret) + return ret; + + WRITE_ONCE(mx_pdev->liveness_enable, val); + + /* Reset watchdog state under sq_lock so a runtime toggle neither inherits a + * stale DEAD verdict on enable nor freezes health the watchdog stops updating. */ + q = mx_pdev->io_queue; + if (q) { + spin_lock_irqsave(&q->sq_lock, flags); + atomic_set(&q->lv_inflight, 0); + if (val) { + WRITE_ONCE(q->lv_progress_jiffies, jiffies); + atomic_set(&q->lv_health, MX_LIVENESS_ALIVE); + } else { + atomic_set(&q->lv_health, MX_LIVENESS_UNKNOWN); + } + spin_unlock_irqrestore(&q->sq_lock, flags); + } + + return count; +} +static DEVICE_ATTR(enable, 0644, liveness_enable_show, liveness_enable_store); + +/* stall_ms/dead_ms/max_mult share one shape: a per-device uint. Store only + * parses; the watchdog and transfer read sites sanitize (1 <= stall < dead, + * mult ceiling), so no bound is enforced here. */ +#define LIVENESS_UINT_ATTR(attr_name, field) \ +static ssize_t attr_name##_show(struct device *dev, \ + struct device_attribute *attr, char *buf) \ +{ \ + struct mx_pci_dev *mx_pdev = dev_get_drvdata(dev); \ + if (!mx_pdev) \ + return -ENODEV; \ + return sysfs_emit(buf, "%u\n", READ_ONCE(mx_pdev->field)); \ +} \ +static ssize_t attr_name##_store(struct device *dev, \ + struct device_attribute *attr, \ + const char *buf, size_t count) \ +{ \ + struct mx_pci_dev *mx_pdev = dev_get_drvdata(dev); \ + unsigned int val; \ + int ret; \ + if (!mx_pdev) \ + return -ENODEV; \ + ret = kstrtouint(buf, 0, &val); \ + if (ret) \ + return ret; \ + WRITE_ONCE(mx_pdev->field, val); \ + return count; \ +} \ +static DEVICE_ATTR(attr_name, 0644, attr_name##_show, attr_name##_store) + +LIVENESS_UINT_ATTR(stall_ms, liveness_stall_ms); +LIVENESS_UINT_ATTR(dead_ms, liveness_dead_ms); +LIVENESS_UINT_ATTR(max_mult, liveness_max_mult); + +static const char * const liveness_health_name[] = { + [MX_LIVENESS_UNKNOWN] = "unknown", + [MX_LIVENESS_ALIVE] = "alive", + [MX_LIVENESS_SUSPECT] = "suspect", + [MX_LIVENESS_DEAD] = "dead", +}; + +/* State attrs report the sentinel (unknown / 0) whenever the watchdog is off or + * the io_queue is gone, never a stale value. */ +static ssize_t health_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct mx_pci_dev *mx_pdev = dev_get_drvdata(dev); + struct mx_queue *q; + int health = MX_LIVENESS_UNKNOWN; + + if (!mx_pdev) + return -ENODEV; + q = mx_pdev->io_queue; + if (READ_ONCE(mx_pdev->liveness_enable) && q) + health = atomic_read(&q->lv_health); + return sysfs_emit(buf, "%s\n", liveness_health_name[health]); +} +static DEVICE_ATTR_RO(health); + +static ssize_t rtt_ns_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct mx_pci_dev *mx_pdev = dev_get_drvdata(dev); + struct mx_queue *q; + u64 rtt = 0; + + if (!mx_pdev) + return -ENODEV; + q = mx_pdev->io_queue; + if (READ_ONCE(mx_pdev->liveness_enable) && q) + rtt = READ_ONCE(q->lv_rtt_ns); + return sysfs_emit(buf, "%llu\n", rtt); +} +static DEVICE_ATTR_RO(rtt_ns); + +static struct attribute *liveness_attrs[] = { + &dev_attr_enable.attr, + &dev_attr_stall_ms.attr, + &dev_attr_dead_ms.attr, + &dev_attr_max_mult.attr, + &dev_attr_health.attr, + &dev_attr_rtt_ns.attr, + NULL, +}; + +static const struct attribute_group liveness_group = { + .name = "liveness", + .attrs = liveness_attrs, +}; + +static const struct attribute_group *mxdma_dev_groups[] = { + &liveness_group, + NULL, +}; + static int create_mx_cdev(struct mx_pci_dev *mx_pdev, int type) { struct mx_char_dev *mx_cdev = &mx_pdev->mx_cdev[type]; @@ -179,7 +321,14 @@ static int create_mx_cdev(struct mx_pci_dev *mx_pdev, int type) return ret; } - dev = device_create(mxdma_class, NULL, mx_cdev->cdev_no, NULL, mx_cdev->cdev.kobj.name); + /* Hang the per-device liveness/ sysfs group off the ioctl node, the device's + * control node; drvdata lets its show/store reach mx_pdev. */ + if (type == MX_CDEV_IOCTL) + dev = device_create_with_groups(mxdma_class, NULL, mx_cdev->cdev_no, + mx_pdev, mxdma_dev_groups, + mx_cdev->cdev.kobj.name); + else + dev = device_create(mxdma_class, NULL, mx_cdev->cdev_no, NULL, mx_cdev->cdev.kobj.name); if (IS_ERR(dev)) { pr_err("Failed to device_created (err=%ld)\n", PTR_ERR(dev)); return PTR_ERR(dev); @@ -287,6 +436,9 @@ static int create_mx_pdev(struct pci_dev *pdev, int cxl_memdev_id) mx_pdev->magic = MAGIC_DEVICE; mx_pdev->pdev = pdev; mx_pdev->dev_id = cxl_memdev_id; + mx_pdev->liveness_stall_ms = LIVENESS_STALL_MS_DEFAULT; + mx_pdev->liveness_dead_ms = LIVENESS_DEAD_MS_DEFAULT; + mx_pdev->liveness_max_mult = LIVENESS_MAX_MULT_DEFAULT; mx_pdev->reserved_hio_qid = -1; mutex_init(&mx_pdev->bar_mmap_lock); diff --git a/ioctl.c b/ioctl.c index 1265f6d..c6ef72b 100644 --- a/ioctl.c +++ b/ioctl.c @@ -120,13 +120,6 @@ struct mx_ioctl_protocol_cmd size_t size; }; -struct mx_ioctl_liveness -{ - uint32_t capability; /* 0 = liveness disabled or FW unsupported */ - uint32_t health; /* enum mx_liveness_health */ - uint64_t rtt_ns; /* last watchdog round-trip, valid when ALIVE */ -}; - #define MX_IOCTL_MAGIC 'X' #define MX_IOCTL_REGISTER_MBOX _IOW(MX_IOCTL_MAGIC, 1, struct mx_ioctl_mbox_info) #define MX_IOCTL_INIT_MBOX _IOW(MX_IOCTL_MAGIC, 2, uint32_t) @@ -138,14 +131,12 @@ struct mx_ioctl_liveness #define MX_IOCTL_PASSTHRU_CMD _IOWR(MX_IOCTL_MAGIC, 8, struct mx_ioctl_passthru_cmd) #define MX_IOCTL_HIO_SEND _IOW(MX_IOCTL_MAGIC, 9, struct mx_ioctl_protocol_cmd) #define MX_IOCTL_HIO_RECV _IOW(MX_IOCTL_MAGIC, 10, struct mx_ioctl_protocol_cmd) -#define MX_IOCTL_GET_LIVENESS _IOR(MX_IOCTL_MAGIC, 11, struct mx_ioctl_liveness) /* MX_IOCTL_* indices must match mx_dma_ioctl_nr_names in trace.h AND both switches in * ioctl_to_device (setup-only early-return + main dispatch). Boundary asserts catch * reorderings, removals, and additions that shift the last NR. */ static_assert(_IOC_NR(MX_IOCTL_REGISTER_MBOX) == 1, "MX_IOCTL_* changed — update trace.h + ioctl_to_device"); static_assert(_IOC_NR(MX_IOCTL_HIO_RECV) == 10, "MX_IOCTL_* changed — update trace.h + ioctl_to_device"); -static_assert(_IOC_NR(MX_IOCTL_GET_LIVENESS) == 11, "MX_IOCTL_* changed — update trace.h + ioctl_to_device"); static uint32_t get_pushable_count(struct mx_mbox *mbox) { @@ -554,25 +545,6 @@ static long ioctl_hio_protocol(struct mx_pci_dev *mx_pdev, unsigned long arg, in return submit_protocol_transfer(mx_pdev, cmd.buf, cmd.size, opcode); } -static long ioctl_get_liveness(struct mx_pci_dev *mx_pdev, unsigned long arg) -{ - struct mx_ioctl_liveness out = {}; - struct mx_queue *q = mx_pdev->io_queue; - - if (!liveness_enable || !q) { - out.capability = 0; - out.health = MX_LIVENESS_UNKNOWN; - } else { - out.capability = 1; - out.health = atomic_read(&q->lv_health); - out.rtt_ns = READ_ONCE(q->lv_rtt_ns); - } - - if (copy_to_user((void __user *)arg, &out, sizeof(out))) - return -EFAULT; - return 0; -} - long ioctl_to_device(struct mx_pci_dev *mx_pdev, unsigned int cmd, unsigned long arg) { long ret; @@ -612,9 +584,6 @@ long ioctl_to_device(struct mx_pci_dev *mx_pdev, unsigned int cmd, unsigned long case MX_IOCTL_HIO_RECV: ret = ioctl_hio_protocol(mx_pdev, arg, IO_OPCODE_RECV); break; - case MX_IOCTL_GET_LIVENESS: - ret = ioctl_get_liveness(mx_pdev, arg); - break; default: pr_warn("unknown ioctl cmd(%u)\n", cmd); ret = -EINVAL; diff --git a/mx_dma.h b/mx_dma.h index 2305735..b08c353 100644 --- a/mx_dma.h +++ b/mx_dma.h @@ -47,6 +47,9 @@ #define LIVENESS_WAIT_CHUNK_MIN_MSEC 50u #define LIVENESS_MAX_MULT_CEIL 1000u +#define LIVENESS_STALL_MS_DEFAULT 1000u +#define LIVENESS_DEAD_MS_DEFAULT 5000u +#define LIVENESS_MAX_MULT_DEFAULT 10u /* * Single-page fast path: embed one struct page * and one scatterlist inside @@ -274,6 +277,7 @@ struct mx_queue_ops { struct mx_queue { struct device *dev; + struct mx_pci_dev *mx_pdev; struct list_head sq_list; spinlock_t sq_lock; atomic_t wait_count; @@ -308,6 +312,13 @@ struct mx_pci_dev { struct pci_dev *pdev; bool enabled; + /* Per-device liveness watchdog config, exposed under the liveness/ sysfs + * group. Off by default; a host service enables it only on ping-capable FW. */ + bool liveness_enable; + unsigned int liveness_stall_ms; + unsigned int liveness_dead_ms; + unsigned int liveness_max_mult; + void __iomem *bar; resource_size_t bar_mapped_size; @@ -458,10 +469,6 @@ static inline void mx_bind_handlers_to_numa(struct mx_pci_dev *mx_pdev) void register_mx_ops_v1(struct mx_operations *ops); void register_mx_ops_v2(struct mx_operations *ops); -extern bool liveness_enable; -extern unsigned int liveness_stall_ms; -extern unsigned int liveness_dead_ms; -extern unsigned int liveness_max_mult; bool is_empty(struct mx_mbox *mbox); bool is_full(struct mx_mbox *mbox); diff --git a/trace.h b/trace.h index 8b51367..6c735ac 100644 --- a/trace.h +++ b/trace.h @@ -47,8 +47,7 @@ { 7, "WRITE_DATA" }, \ { 8, "PASSTHRU_CMD" }, \ { 9, "HIO_SEND" }, \ - { 10, "HIO_RECV" }, \ - { 11, "GET_LIVENESS" } + { 10, "HIO_RECV" } /* --- ioctl entry / exit --- */ diff --git a/transfer.c b/transfer.c index 4d84667..17cceb8 100644 --- a/transfer.c +++ b/transfer.c @@ -408,11 +408,14 @@ static ssize_t mx_transfer_wait(struct mx_pci_dev *mx_pdev, struct mx_transfer * int state; /* Capture id up-front: destroy/release below frees the transfer. */ u32 __maybe_unused xfer_id = (u32)transfer->id; + /* Snapshot the per-device toggle once so the whole wait sees a consistent + * value even if a sysfs write flips it mid-transfer. */ + bool lv_on = READ_ONCE(mx_pdev->liveness_enable); /* Sanitize sysfs-writable params: bounded ceiling multiplier, and a * floored chunk so the wait below never degenerates into a high-frequency * poll. */ - unsigned int mult = liveness_enable ? - clamp(liveness_max_mult, 1u, LIVENESS_MAX_MULT_CEIL) : 1; + unsigned int mult = lv_on ? + clamp(READ_ONCE(mx_pdev->liveness_max_mult), 1u, LIVENESS_MAX_MULT_CEIL) : 1; { /* @@ -421,8 +424,8 @@ static ssize_t mx_transfer_wait(struct mx_pci_dev *mx_pdev, struct mx_transfer * * ceiling of timeout_ms * liveness_max_mult — which also caps a transfer the device silently * dropped while still answering other commands. */ - unsigned int chunk_ms = liveness_enable ? - max(min(liveness_stall_ms, timeout_ms), LIVENESS_WAIT_CHUNK_MIN_MSEC) : timeout_ms; + unsigned int chunk_ms = lv_on ? + max(min(READ_ONCE(mx_pdev->liveness_stall_ms), timeout_ms), LIVENESS_WAIT_CHUNK_MIN_MSEC) : timeout_ms; unsigned long hard_deadline = jiffies + msecs_to_jiffies(timeout_ms) * mult; @@ -431,7 +434,7 @@ static ssize_t mx_transfer_wait(struct mx_pci_dev *mx_pdev, struct mx_transfer * msecs_to_jiffies(chunk_ms)); if (left_time != 0) break; /* completed (>0) or interrupted (<0) */ - if (!liveness_enable) + if (!lv_on) break; /* legacy: single timeout_ms expiry */ if (atomic_read(&mx_pdev->io_queue->lv_health) == MX_LIVENESS_DEAD) break; /* transport dead — fail fast */ @@ -439,7 +442,7 @@ static ssize_t mx_transfer_wait(struct mx_pci_dev *mx_pdev, struct mx_transfer * } if ((long)left_time <= 0) { unsigned long flags; - bool dead = liveness_enable && + bool dead = lv_on && atomic_read(&mx_pdev->io_queue->lv_health) == MX_LIVENESS_DEAD; /* Report DEAD to the caller now; the buffer is still reclaimed lazily * via the zombie path since the device may yet touch it. */