From 86326cda91a33c86491a49d665477348dafc91a1 Mon Sep 17 00:00:00 2001 From: Koa Calloway Date: Wed, 27 May 2026 11:19:23 +0300 Subject: [PATCH 1/7] Add storage operations to et_def.proto --- schema/protobuf/et_def.proto | 169 +++++++++++++++++++++++++++++++++++ 1 file changed, 169 insertions(+) diff --git a/schema/protobuf/et_def.proto b/schema/protobuf/et_def.proto index 965cec86..7e5bb4be 100644 --- a/schema/protobuf/et_def.proto +++ b/schema/protobuf/et_def.proto @@ -114,6 +114,9 @@ enum NodeType { COMM_SEND_NODE = 5; COMM_RECV_NODE = 6; COMM_COLL_NODE = 7; + STORAGE_NODE = 8; + BATCH_NODE = 9; + CHECKPOINT_NODE = 10; } enum CollectiveCommType { @@ -145,6 +148,9 @@ message Node { IOInfo inputs = 8; IOInfo outputs = 9; repeated AttributeProto attr = 10; + + // Populated when type == STORAGE_NODE. See StorageInfo. + optional StorageInfo storage = 11; } message IOInfo { @@ -161,3 +167,166 @@ message Tensor { uint64 elem_bytes = 5; // Number of bytes per element. string device = 6; // Tensor object device location. } + +// ============================================================================= +// Storage operations +// ============================================================================= + +// Layer within the storage stack at which the operation is observed. +enum StorageLevel { + STORAGE_LEVEL_UNKNOWN = 0; + STORAGE_LEVEL_FS = 1; // VFS / filesystem syscalls + STORAGE_LEVEL_PAGE = 2; // page cache / mm + STORAGE_LEVEL_BLOCK = 3; // block layer (bio / request) + STORAGE_LEVEL_DEVICE = 4; // device transport (NVMe/SCSI/ATA) and below; typed schema intentionally omitted +} + +message StorageInfo { + StorageLevel level = 1; + + oneof detail { + FsOp fs = 10; + PageOp page = 11; + BlockOp block = 12; + // STORAGE_LEVEL_DEVICE: no typed detail — use Node.attr. + } +} + +// Filesystem-level operation, modeled on POSIX syscalls and the Linux VFS. +// +// Scope: limited to operations that directly read or write file bytes to +// storage, plus open/close for span context. Metadata-only ops (stat, lseek, +// truncate, fallocate) and ops whose byte traffic surfaces at a lower layer +// (mmap/munmap — see PageOp faults) are intentionally excluded. +// +// References: +// - https://linasm.sourceforge.net/docs/syscalls/filesystem.php +// - https://man7.org/linux/man-pages/dir_section_2.html +// +// int open(const char *pathname, int flags, ... /* mode_t mode */); +// int close(int fd); +// ssize_t read(int fd, void *buf, size_t count); +// ssize_t write(int fd, const void *buf, size_t count); +// ssize_t pread(int fd, void *buf, size_t count, off_t offset); +// ssize_t pwrite(int fd, const void *buf, size_t count, off_t offset); +// int fsync(int fd); +// int fdatasync(int fd); +// int sync_file_range(int fd, off_t offset, off_t nbytes, unsigned int flags); + +message FsOp { + enum Op { + FS_OP_UNKNOWN = 0; + FS_OP_OPEN = 1; + FS_OP_CLOSE = 2; + FS_OP_READ = 3; + FS_OP_WRITE = 4; + FS_OP_PREAD = 5; + FS_OP_PWRITE = 6; + FS_OP_FSYNC = 7; // flush file's dirty data + metadata to device + FS_OP_FDATASYNC = 8; // flush dirty data (+ metadata only if needed for retrieval) + FS_OP_SYNC_FILE_RANGE = 9; // flush a specified byte range of dirty data to device + } + Op op = 1; + string pathname = 2; + int32 fd = 3; + uint64 offset = 4; // byte offset within the file + uint64 size_bytes = 5; // bytes requested / transferred + uint32 flags = 6; +} + +// Page-cache operation. +// +// Scope: limited to page-cache events that are synchronously caused by an +// in-scope FS syscall and that can be causally attributed back to that +// syscall. Background-only events (eviction outside truncate, +// memory-pressure reclaim) and events with no FS syscall upstream (mmap +// page faults) are intentionally excluded. +// +// References: +// - Linux memory management concepts (page cache, writeback): +// https://www.kernel.org/doc/html/latest/admin-guide/mm/concepts.html +// - Tracepoint definitions (authoritative names, fields, print formats): +// filemap: https://elixir.bootlin.com/linux/latest/source/include/trace/events/filemap.h +// writeback: https://elixir.bootlin.com/linux/latest/source/include/trace/events/writeback.h +// - Runtime introspection on a live kernel: +// /sys/kernel/debug/tracing/events/{filemap,writeback}/*/format +// - Readahead kprobe targets (no dedicated tracepoint; kprobed directly): +// page_cache_sync_ra: https://elixir.bootlin.com/linux/latest/A/ident/page_cache_sync_ra +// page_cache_async_ra: https://elixir.bootlin.com/linux/latest/A/ident/page_cache_async_ra +message PageOp { + enum Op { + PAGE_OP_UNKNOWN = 0; + PAGE_OP_ADD_TO_CACHE = 1; // tracepoint: filemap:mm_filemap_add_to_page_cache + PAGE_OP_DIRTY = 2; // tracepoint: writeback:writeback_dirty_folio + // Two readahead variants distinguished by who triggered them. + // SYNC: cache-miss-driven during a read syscall — the user needs these + // pages now, the kernel widens the request to also prefetch ahead. + PAGE_OP_READAHEAD_SYNC = 3; // kprobe: page_cache_sync_ra + // ASYNC: tripwire-driven follow-up — a previous readahead marked a + // page with PG_readahead; when the user touches that page, the + // kernel kicks off the next prefetch without blocking the user. + PAGE_OP_READAHEAD_ASYNC = 6; // kprobe: page_cache_async_ra + } + + // Producers populate whichever of inode / page_index / nr_pages their chosen + // tracepoint provides. + Op op = 1; + uint64 inode = 2; // backing inode when applicable + uint64 page_index = 3; // page offset within the file (PAGE_SIZE units) + uint32 nr_pages = 4; // number of contiguous pages affected +} + +// Block-layer operation (bio / request). +// +// Scope: limited to block-layer ops synchronously caused by an in-scope FS +// syscall via the page layer. Reads (page-cache-miss driven), writes +// (writeback driven by fsync/fdatasync/sync_file_range), and device cache +// flushes are in scope. Discard, secure erase, write-zeroes, and zone +// management originate from filesystem maintenance (trim, fstrim, fallocate) +// or zoned-storage management, not from the in-scope FS syscalls, and are +// intentionally excluded. +// +// References: +// - Linux block layer documentation: +// https://www.kernel.org/doc/html/latest/block/index.html +// - blk-mq / multi-queue block layer: +// https://www.kernel.org/doc/html/latest/block/blk-mq.html +// - REQ_OP_* definitions (include/linux/blk_types.h): +// https://elixir.bootlin.com/linux/latest/source/include/linux/blk_types.h +// - Tracepoint definitions (block lifecycle events Q/G/I/D/C): +// https://elixir.bootlin.com/linux/latest/source/include/trace/events/block.h +// - Runtime introspection on a live kernel: +// /sys/kernel/debug/tracing/events/block/*/format +// - blktrace event semantics (Q/G/I/D/C action identifiers): +// https://man7.org/linux/man-pages/man8/blktrace.8.html +message BlockOp { + // Mirrors REQ_OP_* in include/linux/blk_types.h. The op type is carried by + // every block lifecycle tracepoint (bi_opf & REQ_OP_MASK on the bio); it is + // not itself a distinct tracepoint. + enum Op { + BLOCK_OP_UNKNOWN = 0; + BLOCK_OP_READ = 1; // REQ_OP_READ — issued on page-cache miss inside read/pread + BLOCK_OP_WRITE = 2; // REQ_OP_WRITE — issued by writeback (caused by fsync/fdatasync/sync_file_range) + BLOCK_OP_FLUSH = 3; // REQ_OP_FLUSH — device cache flush (caused by fsync/fdatasync) + } + // Request lifecycle event at which this node was observed. See blktrace(8) + // and include/trace/events/block.h. + enum QueueEvent { + BLOCK_EVENT_UNKNOWN = 0; + BLOCK_EVENT_QUEUE = 1; // Q — tracepoint: block:block_bio_queue (bio submitted via submit_bio) + BLOCK_EVENT_GET = 2; // G — tracepoint: block:block_getrq (request struct allocated) + BLOCK_EVENT_INSERT = 3; // I — tracepoint: block:block_rq_insert (added to scheduler queue) + BLOCK_EVENT_DISPATCH = 4; // D — tracepoint: block:block_rq_issue (handed to driver) + BLOCK_EVENT_COMPLETE = 5; // C — tracepoint: block:block_rq_complete (completion received) + } + + // Producers populate whichever of lba / nr_sectors / req_op_flags their + // chosen tracepoint provides; for FLUSH ops, expect nr_sectors == 0 and + // lba == 0 since flushes carry no data payload. + Op op = 1; + QueueEvent queue_event = 2; + string device = 3; + uint64 lba = 4; // logical block address (in sectors) + uint32 nr_sectors = 5; + uint32 req_op_flags = 6; // REQ_* flags (REQ_SYNC, REQ_FUA, ...) +} From 3126df46170fe4ee3dedf11849b2d4e65d824531 Mon Sep 17 00:00:00 2001 From: Dan Mihailescu Date: Tue, 16 Jun 2026 18:04:10 +0300 Subject: [PATCH 2/7] Rework storage node types into framework-level parents and storage ops Rename BATCH_NODE to DATALOADER_NODE and add KV_TRANSFER_NODE so dataloader, checkpoint, and KV-transfer ops each have a distinct framework-level parent node type, per review feedback. STORAGE_NODE remains the lower-level storage operation emitted under those parents. Co-Authored-By: Claude Opus 4.8 (1M context) --- schema/protobuf/et_def.proto | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/schema/protobuf/et_def.proto b/schema/protobuf/et_def.proto index 7e5bb4be..c21ad35f 100644 --- a/schema/protobuf/et_def.proto +++ b/schema/protobuf/et_def.proto @@ -114,9 +114,13 @@ enum NodeType { COMM_SEND_NODE = 5; COMM_RECV_NODE = 6; COMM_COLL_NODE = 7; + // Lower-level storage operation emitted under a framework-level parent node + // (DATALOADER_NODE, CHECKPOINT_NODE, or KV_TRANSFER_NODE). See StorageInfo. STORAGE_NODE = 8; - BATCH_NODE = 9; + // Framework-level parent nodes that group their constituent STORAGE_NODEs. + DATALOADER_NODE = 9; CHECKPOINT_NODE = 10; + KV_TRANSFER_NODE = 11; } enum CollectiveCommType { From b21f4ea84523d644b3c3877c095ff0b3de59a007 Mon Sep 17 00:00:00 2001 From: Dan Mihailescu Date: Tue, 16 Jun 2026 18:24:19 +0300 Subject: [PATCH 3/7] Rename inner Op enum to OpType in storage op messages Rename the nested Op enum to OpType in FsOp, PageOp, and BlockOp so it no longer reads ambiguously next to the FsOp message name, per review feedback. Co-Authored-By: Claude Opus 4.8 (1M context) --- schema/protobuf/et_def.proto | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/schema/protobuf/et_def.proto b/schema/protobuf/et_def.proto index c21ad35f..59cab729 100644 --- a/schema/protobuf/et_def.proto +++ b/schema/protobuf/et_def.proto @@ -218,7 +218,7 @@ message StorageInfo { // int sync_file_range(int fd, off_t offset, off_t nbytes, unsigned int flags); message FsOp { - enum Op { + enum OpType { FS_OP_UNKNOWN = 0; FS_OP_OPEN = 1; FS_OP_CLOSE = 2; @@ -230,7 +230,7 @@ message FsOp { FS_OP_FDATASYNC = 8; // flush dirty data (+ metadata only if needed for retrieval) FS_OP_SYNC_FILE_RANGE = 9; // flush a specified byte range of dirty data to device } - Op op = 1; + OpType op = 1; string pathname = 2; int32 fd = 3; uint64 offset = 4; // byte offset within the file @@ -258,7 +258,7 @@ message FsOp { // page_cache_sync_ra: https://elixir.bootlin.com/linux/latest/A/ident/page_cache_sync_ra // page_cache_async_ra: https://elixir.bootlin.com/linux/latest/A/ident/page_cache_async_ra message PageOp { - enum Op { + enum OpType { PAGE_OP_UNKNOWN = 0; PAGE_OP_ADD_TO_CACHE = 1; // tracepoint: filemap:mm_filemap_add_to_page_cache PAGE_OP_DIRTY = 2; // tracepoint: writeback:writeback_dirty_folio @@ -274,7 +274,7 @@ message PageOp { // Producers populate whichever of inode / page_index / nr_pages their chosen // tracepoint provides. - Op op = 1; + OpType op = 1; uint64 inode = 2; // backing inode when applicable uint64 page_index = 3; // page offset within the file (PAGE_SIZE units) uint32 nr_pages = 4; // number of contiguous pages affected @@ -307,7 +307,7 @@ message BlockOp { // Mirrors REQ_OP_* in include/linux/blk_types.h. The op type is carried by // every block lifecycle tracepoint (bi_opf & REQ_OP_MASK on the bio); it is // not itself a distinct tracepoint. - enum Op { + enum OpType { BLOCK_OP_UNKNOWN = 0; BLOCK_OP_READ = 1; // REQ_OP_READ — issued on page-cache miss inside read/pread BLOCK_OP_WRITE = 2; // REQ_OP_WRITE — issued by writeback (caused by fsync/fdatasync/sync_file_range) @@ -327,7 +327,7 @@ message BlockOp { // Producers populate whichever of lba / nr_sectors / req_op_flags their // chosen tracepoint provides; for FLUSH ops, expect nr_sectors == 0 and // lba == 0 since flushes carry no data payload. - Op op = 1; + OpType op = 1; QueueEvent queue_event = 2; string device = 3; uint64 lba = 4; // logical block address (in sectors) From ccf0ef51a7f4566cf6c28ebdc3fac68def99174f Mon Sep 17 00:00:00 2001 From: Dan Mihailescu Date: Tue, 16 Jun 2026 19:28:46 +0300 Subject: [PATCH 4/7] Move storage operations into storage.proto Extract StorageLevel, StorageInfo, FsOp, PageOp, and BlockOp into a new storage.proto and import it from et_def.proto, per review feedback (#171). protoc's --python_out emits flat absolute imports between generated modules (et_def_pb2 contains `import storage_pb2`), which fail under the installed chakra.schema.protobuf package. Add schema/protobuf/__init__.py that puts the package directory on sys.path so the generated cross-imports resolve. See protocolbuffers/protobuf#1491. Register storage.proto in setup.cfg (proto_files + grpc_files) and pyproject package-data so it is compiled and shipped alongside et_def.proto. Co-Authored-By: Claude Opus 4.8 (1M context) --- pyproject.toml | 2 +- schema/protobuf/__init__.py | 16 ++++ schema/protobuf/et_def.proto | 165 +-------------------------------- schema/protobuf/storage.proto | 166 ++++++++++++++++++++++++++++++++++ setup.cfg | 8 +- 5 files changed, 191 insertions(+), 166 deletions(-) create mode 100644 schema/protobuf/__init__.py create mode 100644 schema/protobuf/storage.proto diff --git a/pyproject.toml b/pyproject.toml index 99193c03..7123b075 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -38,7 +38,7 @@ Repository = "https://github.com/mlcommons/chakra.git" "chakra.src.visualizer" = "src/visualizer" [tool.setuptools.package-data] -"chakra.schema.protobuf" = ["et_def.proto"] +"chakra.schema.protobuf" = ["et_def.proto", "storage.proto"] [project.scripts] chakra_converter = "chakra.src.converter.converter:main" diff --git a/schema/protobuf/__init__.py b/schema/protobuf/__init__.py new file mode 100644 index 00000000..07c6d42c --- /dev/null +++ b/schema/protobuf/__init__.py @@ -0,0 +1,16 @@ +"""Make the generated ``*_pb2`` modules importable as a package. + +``protoc``'s ``--python_out`` emits flat, absolute imports between generated +modules (e.g. ``et_def_pb2`` contains ``import storage_pb2``). Those resolve +against ``sys.path``, not against this package's own directory, so importing +``chakra.schema.protobuf.et_def_pb2`` would otherwise raise +``ModuleNotFoundError: No module named 'storage_pb2'``. Putting this directory +on ``sys.path`` lets those cross-module imports resolve. + +See https://github.com/protocolbuffers/protobuf/issues/1491. +""" + +import os +import sys + +sys.path.insert(0, os.path.dirname(__file__)) diff --git a/schema/protobuf/et_def.proto b/schema/protobuf/et_def.proto index 59cab729..213312f6 100644 --- a/schema/protobuf/et_def.proto +++ b/schema/protobuf/et_def.proto @@ -2,6 +2,8 @@ syntax = "proto3"; package ChakraProtoMsg; +import "storage.proto"; + message AttributeProto { string name = 1; string doc_string = 2; @@ -171,166 +173,3 @@ message Tensor { uint64 elem_bytes = 5; // Number of bytes per element. string device = 6; // Tensor object device location. } - -// ============================================================================= -// Storage operations -// ============================================================================= - -// Layer within the storage stack at which the operation is observed. -enum StorageLevel { - STORAGE_LEVEL_UNKNOWN = 0; - STORAGE_LEVEL_FS = 1; // VFS / filesystem syscalls - STORAGE_LEVEL_PAGE = 2; // page cache / mm - STORAGE_LEVEL_BLOCK = 3; // block layer (bio / request) - STORAGE_LEVEL_DEVICE = 4; // device transport (NVMe/SCSI/ATA) and below; typed schema intentionally omitted -} - -message StorageInfo { - StorageLevel level = 1; - - oneof detail { - FsOp fs = 10; - PageOp page = 11; - BlockOp block = 12; - // STORAGE_LEVEL_DEVICE: no typed detail — use Node.attr. - } -} - -// Filesystem-level operation, modeled on POSIX syscalls and the Linux VFS. -// -// Scope: limited to operations that directly read or write file bytes to -// storage, plus open/close for span context. Metadata-only ops (stat, lseek, -// truncate, fallocate) and ops whose byte traffic surfaces at a lower layer -// (mmap/munmap — see PageOp faults) are intentionally excluded. -// -// References: -// - https://linasm.sourceforge.net/docs/syscalls/filesystem.php -// - https://man7.org/linux/man-pages/dir_section_2.html -// -// int open(const char *pathname, int flags, ... /* mode_t mode */); -// int close(int fd); -// ssize_t read(int fd, void *buf, size_t count); -// ssize_t write(int fd, const void *buf, size_t count); -// ssize_t pread(int fd, void *buf, size_t count, off_t offset); -// ssize_t pwrite(int fd, const void *buf, size_t count, off_t offset); -// int fsync(int fd); -// int fdatasync(int fd); -// int sync_file_range(int fd, off_t offset, off_t nbytes, unsigned int flags); - -message FsOp { - enum OpType { - FS_OP_UNKNOWN = 0; - FS_OP_OPEN = 1; - FS_OP_CLOSE = 2; - FS_OP_READ = 3; - FS_OP_WRITE = 4; - FS_OP_PREAD = 5; - FS_OP_PWRITE = 6; - FS_OP_FSYNC = 7; // flush file's dirty data + metadata to device - FS_OP_FDATASYNC = 8; // flush dirty data (+ metadata only if needed for retrieval) - FS_OP_SYNC_FILE_RANGE = 9; // flush a specified byte range of dirty data to device - } - OpType op = 1; - string pathname = 2; - int32 fd = 3; - uint64 offset = 4; // byte offset within the file - uint64 size_bytes = 5; // bytes requested / transferred - uint32 flags = 6; -} - -// Page-cache operation. -// -// Scope: limited to page-cache events that are synchronously caused by an -// in-scope FS syscall and that can be causally attributed back to that -// syscall. Background-only events (eviction outside truncate, -// memory-pressure reclaim) and events with no FS syscall upstream (mmap -// page faults) are intentionally excluded. -// -// References: -// - Linux memory management concepts (page cache, writeback): -// https://www.kernel.org/doc/html/latest/admin-guide/mm/concepts.html -// - Tracepoint definitions (authoritative names, fields, print formats): -// filemap: https://elixir.bootlin.com/linux/latest/source/include/trace/events/filemap.h -// writeback: https://elixir.bootlin.com/linux/latest/source/include/trace/events/writeback.h -// - Runtime introspection on a live kernel: -// /sys/kernel/debug/tracing/events/{filemap,writeback}/*/format -// - Readahead kprobe targets (no dedicated tracepoint; kprobed directly): -// page_cache_sync_ra: https://elixir.bootlin.com/linux/latest/A/ident/page_cache_sync_ra -// page_cache_async_ra: https://elixir.bootlin.com/linux/latest/A/ident/page_cache_async_ra -message PageOp { - enum OpType { - PAGE_OP_UNKNOWN = 0; - PAGE_OP_ADD_TO_CACHE = 1; // tracepoint: filemap:mm_filemap_add_to_page_cache - PAGE_OP_DIRTY = 2; // tracepoint: writeback:writeback_dirty_folio - // Two readahead variants distinguished by who triggered them. - // SYNC: cache-miss-driven during a read syscall — the user needs these - // pages now, the kernel widens the request to also prefetch ahead. - PAGE_OP_READAHEAD_SYNC = 3; // kprobe: page_cache_sync_ra - // ASYNC: tripwire-driven follow-up — a previous readahead marked a - // page with PG_readahead; when the user touches that page, the - // kernel kicks off the next prefetch without blocking the user. - PAGE_OP_READAHEAD_ASYNC = 6; // kprobe: page_cache_async_ra - } - - // Producers populate whichever of inode / page_index / nr_pages their chosen - // tracepoint provides. - OpType op = 1; - uint64 inode = 2; // backing inode when applicable - uint64 page_index = 3; // page offset within the file (PAGE_SIZE units) - uint32 nr_pages = 4; // number of contiguous pages affected -} - -// Block-layer operation (bio / request). -// -// Scope: limited to block-layer ops synchronously caused by an in-scope FS -// syscall via the page layer. Reads (page-cache-miss driven), writes -// (writeback driven by fsync/fdatasync/sync_file_range), and device cache -// flushes are in scope. Discard, secure erase, write-zeroes, and zone -// management originate from filesystem maintenance (trim, fstrim, fallocate) -// or zoned-storage management, not from the in-scope FS syscalls, and are -// intentionally excluded. -// -// References: -// - Linux block layer documentation: -// https://www.kernel.org/doc/html/latest/block/index.html -// - blk-mq / multi-queue block layer: -// https://www.kernel.org/doc/html/latest/block/blk-mq.html -// - REQ_OP_* definitions (include/linux/blk_types.h): -// https://elixir.bootlin.com/linux/latest/source/include/linux/blk_types.h -// - Tracepoint definitions (block lifecycle events Q/G/I/D/C): -// https://elixir.bootlin.com/linux/latest/source/include/trace/events/block.h -// - Runtime introspection on a live kernel: -// /sys/kernel/debug/tracing/events/block/*/format -// - blktrace event semantics (Q/G/I/D/C action identifiers): -// https://man7.org/linux/man-pages/man8/blktrace.8.html -message BlockOp { - // Mirrors REQ_OP_* in include/linux/blk_types.h. The op type is carried by - // every block lifecycle tracepoint (bi_opf & REQ_OP_MASK on the bio); it is - // not itself a distinct tracepoint. - enum OpType { - BLOCK_OP_UNKNOWN = 0; - BLOCK_OP_READ = 1; // REQ_OP_READ — issued on page-cache miss inside read/pread - BLOCK_OP_WRITE = 2; // REQ_OP_WRITE — issued by writeback (caused by fsync/fdatasync/sync_file_range) - BLOCK_OP_FLUSH = 3; // REQ_OP_FLUSH — device cache flush (caused by fsync/fdatasync) - } - // Request lifecycle event at which this node was observed. See blktrace(8) - // and include/trace/events/block.h. - enum QueueEvent { - BLOCK_EVENT_UNKNOWN = 0; - BLOCK_EVENT_QUEUE = 1; // Q — tracepoint: block:block_bio_queue (bio submitted via submit_bio) - BLOCK_EVENT_GET = 2; // G — tracepoint: block:block_getrq (request struct allocated) - BLOCK_EVENT_INSERT = 3; // I — tracepoint: block:block_rq_insert (added to scheduler queue) - BLOCK_EVENT_DISPATCH = 4; // D — tracepoint: block:block_rq_issue (handed to driver) - BLOCK_EVENT_COMPLETE = 5; // C — tracepoint: block:block_rq_complete (completion received) - } - - // Producers populate whichever of lba / nr_sectors / req_op_flags their - // chosen tracepoint provides; for FLUSH ops, expect nr_sectors == 0 and - // lba == 0 since flushes carry no data payload. - OpType op = 1; - QueueEvent queue_event = 2; - string device = 3; - uint64 lba = 4; // logical block address (in sectors) - uint32 nr_sectors = 5; - uint32 req_op_flags = 6; // REQ_* flags (REQ_SYNC, REQ_FUA, ...) -} diff --git a/schema/protobuf/storage.proto b/schema/protobuf/storage.proto new file mode 100644 index 00000000..d053d0d3 --- /dev/null +++ b/schema/protobuf/storage.proto @@ -0,0 +1,166 @@ +syntax = "proto3"; + +package ChakraProtoMsg; + +// ============================================================================= +// Storage operations +// ============================================================================= + +// Layer within the storage stack at which the operation is observed. +enum StorageLevel { + STORAGE_LEVEL_UNKNOWN = 0; + STORAGE_LEVEL_FS = 1; // VFS / filesystem syscalls + STORAGE_LEVEL_PAGE = 2; // page cache / mm + STORAGE_LEVEL_BLOCK = 3; // block layer (bio / request) + STORAGE_LEVEL_DEVICE = 4; // device transport (NVMe/SCSI/ATA) and below; typed schema intentionally omitted +} + +message StorageInfo { + StorageLevel level = 1; + + oneof detail { + FsOp fs = 10; + PageOp page = 11; + BlockOp block = 12; + // STORAGE_LEVEL_DEVICE: no typed detail — use Node.attr. + } +} + +// Filesystem-level operation, modeled on POSIX syscalls and the Linux VFS. +// +// Scope: limited to operations that directly read or write file bytes to +// storage, plus open/close for span context. Metadata-only ops (stat, lseek, +// truncate, fallocate) and ops whose byte traffic surfaces at a lower layer +// (mmap/munmap — see PageOp faults) are intentionally excluded. +// +// References: +// - https://linasm.sourceforge.net/docs/syscalls/filesystem.php +// - https://man7.org/linux/man-pages/dir_section_2.html +// +// int open(const char *pathname, int flags, ... /* mode_t mode */); +// int close(int fd); +// ssize_t read(int fd, void *buf, size_t count); +// ssize_t write(int fd, const void *buf, size_t count); +// ssize_t pread(int fd, void *buf, size_t count, off_t offset); +// ssize_t pwrite(int fd, const void *buf, size_t count, off_t offset); +// int fsync(int fd); +// int fdatasync(int fd); +// int sync_file_range(int fd, off_t offset, off_t nbytes, unsigned int flags); + +message FsOp { + enum OpType { + FS_OP_UNKNOWN = 0; + FS_OP_OPEN = 1; + FS_OP_CLOSE = 2; + FS_OP_READ = 3; + FS_OP_WRITE = 4; + FS_OP_PREAD = 5; + FS_OP_PWRITE = 6; + FS_OP_FSYNC = 7; // flush file's dirty data + metadata to device + FS_OP_FDATASYNC = 8; // flush dirty data (+ metadata only if needed for retrieval) + FS_OP_SYNC_FILE_RANGE = 9; // flush a specified byte range of dirty data to device + } + OpType op = 1; + string pathname = 2; + int32 fd = 3; + uint64 offset = 4; // byte offset within the file + uint64 size_bytes = 5; // bytes requested / transferred + uint32 flags = 6; +} + +// Page-cache operation. +// +// Scope: limited to page-cache events that are synchronously caused by an +// in-scope FS syscall and that can be causally attributed back to that +// syscall. Background-only events (eviction outside truncate, +// memory-pressure reclaim) and events with no FS syscall upstream (mmap +// page faults) are intentionally excluded. +// +// References: +// - Linux memory management concepts (page cache, writeback): +// https://www.kernel.org/doc/html/latest/admin-guide/mm/concepts.html +// - Tracepoint definitions (authoritative names, fields, print formats): +// filemap: https://elixir.bootlin.com/linux/latest/source/include/trace/events/filemap.h +// writeback: https://elixir.bootlin.com/linux/latest/source/include/trace/events/writeback.h +// - Runtime introspection on a live kernel: +// /sys/kernel/debug/tracing/events/{filemap,writeback}/*/format +// - Readahead kprobe targets (no dedicated tracepoint; kprobed directly): +// page_cache_sync_ra: https://elixir.bootlin.com/linux/latest/A/ident/page_cache_sync_ra +// page_cache_async_ra: https://elixir.bootlin.com/linux/latest/A/ident/page_cache_async_ra +message PageOp { + enum OpType { + PAGE_OP_UNKNOWN = 0; + PAGE_OP_ADD_TO_CACHE = 1; // tracepoint: filemap:mm_filemap_add_to_page_cache + PAGE_OP_DIRTY = 2; // tracepoint: writeback:writeback_dirty_folio + // Two readahead variants distinguished by who triggered them. + // SYNC: cache-miss-driven during a read syscall — the user needs these + // pages now, the kernel widens the request to also prefetch ahead. + PAGE_OP_READAHEAD_SYNC = 3; // kprobe: page_cache_sync_ra + // ASYNC: tripwire-driven follow-up — a previous readahead marked a + // page with PG_readahead; when the user touches that page, the + // kernel kicks off the next prefetch without blocking the user. + PAGE_OP_READAHEAD_ASYNC = 6; // kprobe: page_cache_async_ra + } + + // Producers populate whichever of inode / page_index / nr_pages their chosen + // tracepoint provides. + OpType op = 1; + uint64 inode = 2; // backing inode when applicable + uint64 page_index = 3; // page offset within the file (PAGE_SIZE units) + uint32 nr_pages = 4; // number of contiguous pages affected +} + +// Block-layer operation (bio / request). +// +// Scope: limited to block-layer ops synchronously caused by an in-scope FS +// syscall via the page layer. Reads (page-cache-miss driven), writes +// (writeback driven by fsync/fdatasync/sync_file_range), and device cache +// flushes are in scope. Discard, secure erase, write-zeroes, and zone +// management originate from filesystem maintenance (trim, fstrim, fallocate) +// or zoned-storage management, not from the in-scope FS syscalls, and are +// intentionally excluded. +// +// References: +// - Linux block layer documentation: +// https://www.kernel.org/doc/html/latest/block/index.html +// - blk-mq / multi-queue block layer: +// https://www.kernel.org/doc/html/latest/block/blk-mq.html +// - REQ_OP_* definitions (include/linux/blk_types.h): +// https://elixir.bootlin.com/linux/latest/source/include/linux/blk_types.h +// - Tracepoint definitions (block lifecycle events Q/G/I/D/C): +// https://elixir.bootlin.com/linux/latest/source/include/trace/events/block.h +// - Runtime introspection on a live kernel: +// /sys/kernel/debug/tracing/events/block/*/format +// - blktrace event semantics (Q/G/I/D/C action identifiers): +// https://man7.org/linux/man-pages/man8/blktrace.8.html +message BlockOp { + // Mirrors REQ_OP_* in include/linux/blk_types.h. The op type is carried by + // every block lifecycle tracepoint (bi_opf & REQ_OP_MASK on the bio); it is + // not itself a distinct tracepoint. + enum OpType { + BLOCK_OP_UNKNOWN = 0; + BLOCK_OP_READ = 1; // REQ_OP_READ — issued on page-cache miss inside read/pread + BLOCK_OP_WRITE = 2; // REQ_OP_WRITE — issued by writeback (caused by fsync/fdatasync/sync_file_range) + BLOCK_OP_FLUSH = 3; // REQ_OP_FLUSH — device cache flush (caused by fsync/fdatasync) + } + // Request lifecycle event at which this node was observed. See blktrace(8) + // and include/trace/events/block.h. + enum QueueEvent { + BLOCK_EVENT_UNKNOWN = 0; + BLOCK_EVENT_QUEUE = 1; // Q — tracepoint: block:block_bio_queue (bio submitted via submit_bio) + BLOCK_EVENT_GET = 2; // G — tracepoint: block:block_getrq (request struct allocated) + BLOCK_EVENT_INSERT = 3; // I — tracepoint: block:block_rq_insert (added to scheduler queue) + BLOCK_EVENT_DISPATCH = 4; // D — tracepoint: block:block_rq_issue (handed to driver) + BLOCK_EVENT_COMPLETE = 5; // C — tracepoint: block:block_rq_complete (completion received) + } + + // Producers populate whichever of lba / nr_sectors / req_op_flags their + // chosen tracepoint provides; for FLUSH ops, expect nr_sectors == 0 and + // lba == 0 since flushes carry no data payload. + OpType op = 1; + QueueEvent queue_event = 2; + string device = 3; + uint64 lba = 4; // logical block address (in sectors) + uint32 nr_sectors = 5; + uint32 req_op_flags = 6; // REQ_* flags (REQ_SYNC, REQ_FUA, ...) +} diff --git a/setup.cfg b/setup.cfg index 72fe550a..09f1134f 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,5 +1,9 @@ [build_grpc] -proto_files = et_def.proto -grpc_files = et_def.proto +proto_files = + et_def.proto + storage.proto +grpc_files = + et_def.proto + storage.proto proto_path = schema/protobuf/ output_path = schema/protobuf/ From 4326f4169c855769c3908f9bf12db4975a5a2e4f Mon Sep 17 00:00:00 2001 From: Dan Mihailescu Date: Wed, 17 Jun 2026 06:52:41 +0300 Subject: [PATCH 5/7] Drop parent/child node-type comments from NodeType enum Remove the comments coupling STORAGE_NODE to the framework-level parent nodes: tracing may capture only framework-level nodes, or only storage operations, so the asserted relationship is not mandatory. Per review feedback. Co-Authored-By: Claude Opus 4.8 (1M context) --- schema/protobuf/et_def.proto | 3 --- 1 file changed, 3 deletions(-) diff --git a/schema/protobuf/et_def.proto b/schema/protobuf/et_def.proto index 213312f6..efed590f 100644 --- a/schema/protobuf/et_def.proto +++ b/schema/protobuf/et_def.proto @@ -116,10 +116,7 @@ enum NodeType { COMM_SEND_NODE = 5; COMM_RECV_NODE = 6; COMM_COLL_NODE = 7; - // Lower-level storage operation emitted under a framework-level parent node - // (DATALOADER_NODE, CHECKPOINT_NODE, or KV_TRANSFER_NODE). See StorageInfo. STORAGE_NODE = 8; - // Framework-level parent nodes that group their constituent STORAGE_NODEs. DATALOADER_NODE = 9; CHECKPOINT_NODE = 10; KV_TRANSFER_NODE = 11; From 5b24e2797b33273f7476c0ea7a59ec9ba6768cbe Mon Sep 17 00:00:00 2001 From: Koa Calloway Date: Mon, 6 Jul 2026 21:50:09 -0700 Subject: [PATCH 6/7] Build all protos in feeder tests and CodeQL CI workflows. Start schema/protobuf/__init__.py docstring summary on the second line to satisfy ruff D213. --- .github/workflows/codeql.yml | 5 +++-- .github/workflows/feeder_tests.yml | 24 ++++++++++++++++-------- schema/protobuf/__init__.py | 3 ++- 3 files changed, 21 insertions(+), 11 deletions(-) diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index f7c89458..b0ad3fc6 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -87,10 +87,11 @@ jobs: SCRIPT_DIR=. BUILD_DIR="${SCRIPT_DIR:?}"/build CHAKRA_ET_DIR="${SCRIPT_DIR:?}"/schema/protobuf - protoc et_def.proto \ + + protoc "${CHAKRA_ET_DIR:?}"/*.proto \ --proto_path="${CHAKRA_ET_DIR:?}" \ --cpp_out="${CHAKRA_ET_DIR:?}" - g++ -shared -fPIC -Wall src/feeder/et_feeder.cpp src/feeder/et_feeder_node.cpp src/third_party/utils/protoio.cc schema/protobuf/et_def.pb.cc -o libfeeder.so -lprotobuf -I . -I src/feeder -I src/third_party/utils -I schema/protobuf + g++ -shared -fPIC -Wall src/feeder/et_feeder.cpp src/feeder/et_feeder_node.cpp src/third_party/utils/protoio.cc schema/protobuf/*.pb.cc -o libfeeder.so -lprotobuf -I . -I src/feeder -I src/third_party/utils -I schema/protobuf - name: Perform CodeQL Analysis uses: github/codeql-action/analyze@v3 diff --git a/.github/workflows/feeder_tests.yml b/.github/workflows/feeder_tests.yml index a369747a..661da853 100644 --- a/.github/workflows/feeder_tests.yml +++ b/.github/workflows/feeder_tests.yml @@ -24,14 +24,22 @@ jobs: SCRIPT_DIR=. BUILD_DIR="${SCRIPT_DIR:?}"/build CHAKRA_ET_DIR="${SCRIPT_DIR:?}"/schema/protobuf - protoc et_def.proto \ + INCLUDES="-I src/third_party/utils -I schema/protobuf -I src/feeder" + + protoc "${CHAKRA_ET_DIR:?}"/*.proto \ --proto_path="${CHAKRA_ET_DIR:?}" \ --cpp_out="${CHAKRA_ET_DIR:?}" - g++ -Wall -I src/third_party/utils -I schema/protobuf -I src/feeder -c schema/protobuf/et_def.pb.cc -o schema/protobuf/et_def.pb.o - g++ -Wall -I src/third_party/utils -I schema/protobuf -I src/feeder -c src/feeder/et_feeder.cpp -o src/feeder/et_feeder.o - g++ -Wall -I src/third_party/utils -I schema/protobuf -I src/feeder -c src/feeder/et_feeder_node.cpp -o src/feeder/et_feeder_node.o - g++ -Wall -I src/third_party/utils -I schema/protobuf -I src/feeder -c src/third_party/utils/protoio.cc -o src/third_party/utils/protoio.o - g++ -Wall -I src/third_party/utils -I schema/protobuf -I src/feeder -c tests/feeder/tests.cpp -o tests/feeder/tests.o - g++ -Wall -I src/third_party/utils -I schema/protobuf -I src/feeder -o feeder_tests schema/protobuf/et_def.pb.o src/feeder/et_feeder.o src/feeder/et_feeder_node.o src/third_party/utils/protoio.o tests/feeder/tests.o -lgtest -lgtest_main -lprotobuf -lpthread + + PB_OBJECTS="" + for cc in "${CHAKRA_ET_DIR:?}"/*.pb.cc; do + obj="${cc%.cc}.o" + g++ -Wall ${INCLUDES} -c "${cc}" -o "${obj}" + PB_OBJECTS="${PB_OBJECTS} ${obj}" + done + g++ -Wall ${INCLUDES} -c src/feeder/et_feeder.cpp -o src/feeder/et_feeder.o + g++ -Wall ${INCLUDES} -c src/feeder/et_feeder_node.cpp -o src/feeder/et_feeder_node.o + g++ -Wall ${INCLUDES} -c src/third_party/utils/protoio.cc -o src/third_party/utils/protoio.o + g++ -Wall ${INCLUDES} -c tests/feeder/tests.cpp -o tests/feeder/tests.o + g++ -Wall ${INCLUDES} -o feeder_tests ${PB_OBJECTS} src/feeder/et_feeder.o src/feeder/et_feeder_node.o src/third_party/utils/protoio.o tests/feeder/tests.o -lgtest -lgtest_main -lprotobuf -lpthread - name: Run tests - run: ./feeder_tests \ No newline at end of file + run: ./feeder_tests diff --git a/schema/protobuf/__init__.py b/schema/protobuf/__init__.py index 07c6d42c..f6079941 100644 --- a/schema/protobuf/__init__.py +++ b/schema/protobuf/__init__.py @@ -1,4 +1,5 @@ -"""Make the generated ``*_pb2`` modules importable as a package. +""" +Make the generated ``*_pb2`` modules importable as a package. ``protoc``'s ``--python_out`` emits flat, absolute imports between generated modules (e.g. ``et_def_pb2`` contains ``import storage_pb2``). Those resolve From 9242ebd48aabb18eeb3471fcf29b66427f3f2723 Mon Sep 17 00:00:00 2001 From: Koa Calloway Date: Mon, 6 Jul 2026 22:27:04 -0700 Subject: [PATCH 7/7] Consolidate framework-level node types into DATA_OP_NODE. Use op_info oneof to store either StorageInfo or DataOpType. --- schema/protobuf/et_def.proto | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/schema/protobuf/et_def.proto b/schema/protobuf/et_def.proto index efed590f..add439c8 100644 --- a/schema/protobuf/et_def.proto +++ b/schema/protobuf/et_def.proto @@ -117,9 +117,14 @@ enum NodeType { COMM_RECV_NODE = 6; COMM_COLL_NODE = 7; STORAGE_NODE = 8; - DATALOADER_NODE = 9; - CHECKPOINT_NODE = 10; - KV_TRANSFER_NODE = 11; + DATA_OP_NODE = 9; +} + +enum DataOpType { + DATA_OP_UNKNOWN = 0; + DATA_OP_DATALOADER = 1; + DATA_OP_CHECKPOINT = 2; + DATA_OP_KV_TRANSFER = 3; } enum CollectiveCommType { @@ -152,8 +157,11 @@ message Node { IOInfo outputs = 9; repeated AttributeProto attr = 10; - // Populated when type == STORAGE_NODE. See StorageInfo. - optional StorageInfo storage = 11; + // Type-specific info. + oneof op_info { + StorageInfo storage = 11; // type == STORAGE_NODE + DataOpType data_op = 12; // type == DATA_OP_NODE + } } message IOInfo {