From 2951a9cf759247a65aa880447c2ccf61d85ba771 Mon Sep 17 00:00:00 2001 From: Max Schettler Date: Sat, 18 Jul 2026 09:09:36 +0200 Subject: [PATCH 01/25] Consistently crash task runners on DB failure This is to make failures visible and avoid any chance for latently lingering broken connections which may occur in FDB. This relies on the deployment mechanism to a) monitor the failures, and b) restart the task runner. --- .../services/tasks_runner_backup.py | 6 --- .../services/tasks_runner_batch_migration.py | 7 +-- .../services/tasks_runner_cluster_expand.py | 7 +-- .../services/tasks_runner_failed_migration.py | 6 --- .../services/tasks_runner_jc_comp.py | 6 --- .../services/tasks_runner_lvol_migration.py | 6 --- .../services/tasks_runner_migration.py | 6 --- .../tasks_runner_new_dev_migration.py | 6 --- .../services/tasks_runner_node_add.py | 16 +------ .../services/tasks_runner_node_removal.py | 6 +-- .../services/tasks_runner_port_allow.py | 6 --- .../tasks_runner_replication_final.py | 7 +-- .../services/tasks_runner_restart.py | 6 --- .../services/tasks_runner_sync_lvol_del.py | 47 ++----------------- 14 files changed, 9 insertions(+), 129 deletions(-) diff --git a/simplyblock_core/services/tasks_runner_backup.py b/simplyblock_core/services/tasks_runner_backup.py index 8f0ed8af05..dfb1a5e49c 100644 --- a/simplyblock_core/services/tasks_runner_backup.py +++ b/simplyblock_core/services/tasks_runner_backup.py @@ -449,12 +449,6 @@ def process_task(task, cl): if __name__ == "__main__": logger.info("Starting backup tasks runner...") while True: - try: - db.get_clusters() - except Exception as e: - logger.error(f"Failed to get clusters: {e}") - time.sleep(3) - continue clusters = db.get_clusters() for cl in clusters: if cl.status == Cluster.STATUS_IN_ACTIVATION: diff --git a/simplyblock_core/services/tasks_runner_batch_migration.py b/simplyblock_core/services/tasks_runner_batch_migration.py index 5e45b6f724..3015ec8b40 100644 --- a/simplyblock_core/services/tasks_runner_batch_migration.py +++ b/simplyblock_core/services/tasks_runner_batch_migration.py @@ -1084,12 +1084,7 @@ def main(): logger.info("Starting Batch Migration orchestrator task runner...") while True: - try: - clusters = db.get_clusters() - except Exception as e: - logger.error(f"Failed to get clusters: {e}") - time.sleep(3) - continue + clusters = db.get_clusters() if not clusters: logger.error("No clusters found!") diff --git a/simplyblock_core/services/tasks_runner_cluster_expand.py b/simplyblock_core/services/tasks_runner_cluster_expand.py index c9c17af6dc..03a3a4e6e6 100644 --- a/simplyblock_core/services/tasks_runner_cluster_expand.py +++ b/simplyblock_core/services/tasks_runner_cluster_expand.py @@ -101,12 +101,7 @@ def process_task(task): def main(): logger.info("Starting Tasks runner cluster expand...") while True: - try: - clusters = db.get_clusters() - except Exception as e: - logger.error(f"Failed to get clusters: {e}") - time.sleep(3) - continue + clusters = db.get_clusters() if not clusters: logger.error("No clusters found!") else: diff --git a/simplyblock_core/services/tasks_runner_failed_migration.py b/simplyblock_core/services/tasks_runner_failed_migration.py index 4b591d3b22..6ea8cf5f79 100644 --- a/simplyblock_core/services/tasks_runner_failed_migration.py +++ b/simplyblock_core/services/tasks_runner_failed_migration.py @@ -140,12 +140,6 @@ def task_runner(task): def main(): logger.info("Starting Tasks runner...") while True: - try: - db.get_clusters() - except Exception as e: - logger.error(f"Failed to get clusters: {e}") - time.sleep(3) - continue time.sleep(3) clusters = db.get_clusters() if not clusters: diff --git a/simplyblock_core/services/tasks_runner_jc_comp.py b/simplyblock_core/services/tasks_runner_jc_comp.py index 64580da519..b325b51365 100644 --- a/simplyblock_core/services/tasks_runner_jc_comp.py +++ b/simplyblock_core/services/tasks_runner_jc_comp.py @@ -18,12 +18,6 @@ def main(): logger.info("Starting Tasks runner...") while True: - try: - db.get_clusters() - except Exception as e: - logger.error(f"Failed to get clusters: {e}") - time.sleep(3) - continue clusters = db.get_clusters() if not clusters: logger.error("No clusters found!") diff --git a/simplyblock_core/services/tasks_runner_lvol_migration.py b/simplyblock_core/services/tasks_runner_lvol_migration.py index e5c2a1d687..769d9c0271 100644 --- a/simplyblock_core/services/tasks_runner_lvol_migration.py +++ b/simplyblock_core/services/tasks_runner_lvol_migration.py @@ -3721,12 +3721,6 @@ def _cancel_stale_new_migrations(cluster_id): logger.info("Starting LVol Migration task runner...") while True: - try: - db.get_clusters() - except Exception as e: - logger.error(f"Failed to get clusters: {e}") - time.sleep(3) - continue clusters = db.get_clusters() if not clusters: logger.error("No clusters found!") diff --git a/simplyblock_core/services/tasks_runner_migration.py b/simplyblock_core/services/tasks_runner_migration.py index c9a191d606..e2f645c27a 100644 --- a/simplyblock_core/services/tasks_runner_migration.py +++ b/simplyblock_core/services/tasks_runner_migration.py @@ -268,12 +268,6 @@ def _set_master_task_status(master_task, status): def main(): logger.info("Starting Tasks runner...") while True: - try: - db.get_clusters() - except Exception as e: - logger.error(f"Failed to get clusters: {e}") - time.sleep(3) - continue clusters = db.get_clusters() if not clusters: logger.error("No clusters found!") diff --git a/simplyblock_core/services/tasks_runner_new_dev_migration.py b/simplyblock_core/services/tasks_runner_new_dev_migration.py index a7324c78c6..76b611e59e 100644 --- a/simplyblock_core/services/tasks_runner_new_dev_migration.py +++ b/simplyblock_core/services/tasks_runner_new_dev_migration.py @@ -168,12 +168,6 @@ def task_runner(task): def main(): logger.info("Starting Tasks runner...") while True: - try: - db.get_clusters() - except Exception as e: - logger.error(f"Failed to get clusters: {e}") - time.sleep(3) - continue time.sleep(3) clusters = db.get_clusters() if not clusters: diff --git a/simplyblock_core/services/tasks_runner_node_add.py b/simplyblock_core/services/tasks_runner_node_add.py index a6836bff53..224690b163 100644 --- a/simplyblock_core/services/tasks_runner_node_add.py +++ b/simplyblock_core/services/tasks_runner_node_add.py @@ -203,24 +203,12 @@ def main(): executor = ThreadPoolExecutor(max_workers=MAX_CONCURRENT_NODE_ADDS) while True: - try: - clusters = db.get_clusters() - except Exception as e: - logger.error(f"Failed to get clusters: {e}") - time.sleep(3) - continue + clusters = db.get_clusters() if not clusters: logger.error("No clusters found!") else: for cl in clusters: - # An unhandled FDBError here (1031 transaction timeout) killed - # this runner at cluster start on 2026-07-16 — no auto-restart - # ran for the rest of the run. Log and retry next tick instead. - try: - tasks = db.get_job_tasks(cl.get_id(), reverse=False) - except Exception as e: - logger.error(f"Failed to read tasks for cluster {cl.get_id()}: {e}") - continue + tasks = db.get_job_tasks(cl.get_id(), reverse=False) for task in tasks: if task.function_name != JobSchedule.FN_NODE_ADD: continue diff --git a/simplyblock_core/services/tasks_runner_node_removal.py b/simplyblock_core/services/tasks_runner_node_removal.py index a90fbc36e5..412cb4dcaa 100644 --- a/simplyblock_core/services/tasks_runner_node_removal.py +++ b/simplyblock_core/services/tasks_runner_node_removal.py @@ -70,11 +70,7 @@ def process_task(task): while True: time.sleep(constants.TASK_EXEC_INTERVAL_SEC) - try: - clusters = db.get_clusters() - except Exception as e: - logger.error(f"Failed to get clusters: {e}") - continue + clusters = db.get_clusters() if not clusters: logger.error("No clusters found!") continue diff --git a/simplyblock_core/services/tasks_runner_port_allow.py b/simplyblock_core/services/tasks_runner_port_allow.py index bd8f4f8530..6868dcbe99 100644 --- a/simplyblock_core/services/tasks_runner_port_allow.py +++ b/simplyblock_core/services/tasks_runner_port_allow.py @@ -1068,12 +1068,6 @@ def _fail(msg): def _main(): logger.info("Starting Tasks runner...") while True: - try: - db.get_clusters() - except Exception as e: - logger.error(f"Failed to get clusters: {e}") - time.sleep(3) - continue clusters = db.get_clusters() if not clusters: logger.error("No clusters found!") diff --git a/simplyblock_core/services/tasks_runner_replication_final.py b/simplyblock_core/services/tasks_runner_replication_final.py index 6e21839450..f9b4837875 100644 --- a/simplyblock_core/services/tasks_runner_replication_final.py +++ b/simplyblock_core/services/tasks_runner_replication_final.py @@ -279,12 +279,7 @@ def _prepare_cutover(task, lvol, src_node, tgt_node): def main(): logger.info("Starting replication-final tasks runner...") while True: - try: - clusters = db.get_clusters() - except Exception as e: - logger.error(f"Failed to get clusters: {e}") - time.sleep(3) - continue + clusters = db.get_clusters() for cl in clusters: for task in db.get_job_tasks(cl.get_id(), reverse=False): if task.function_name != JobSchedule.FN_REPLICATION_FINAL: diff --git a/simplyblock_core/services/tasks_runner_restart.py b/simplyblock_core/services/tasks_runner_restart.py index 82d3a75497..d90df63a74 100644 --- a/simplyblock_core/services/tasks_runner_restart.py +++ b/simplyblock_core/services/tasks_runner_restart.py @@ -827,12 +827,6 @@ def _process_restart_task(task_uuid): def main(): logger.info("Starting Tasks runner...") while True: - try: - db.get_clusters() - except Exception as e: - logger.error(f"Failed to get clusters: {e}") - time.sleep(3) - continue clusters = db.get_clusters() if not clusters: logger.error("No clusters found!") diff --git a/simplyblock_core/services/tasks_runner_sync_lvol_del.py b/simplyblock_core/services/tasks_runner_sync_lvol_del.py index 4e8fd85583..99b16cd197 100644 --- a/simplyblock_core/services/tasks_runner_sync_lvol_del.py +++ b/simplyblock_core/services/tasks_runner_sync_lvol_del.py @@ -1,5 +1,4 @@ # coding=utf-8 -import sys import time from typing import Optional @@ -24,17 +23,6 @@ def get_primary_node(task) -> Optional[StorageNode]: return None -# A persistent FDB read failure — or an unexpectedly empty cluster list — on this -# long-lived process means the FDB client is wedged: fdb.open() caches the -# Database and the network thread is per-process, so re-reading the same handle -# never recovers; only a fresh process does. Incident -# mass_create_delete_docker-20260629: this runner sat idle ~8h logging -# "No clusters found!" after a startup FDBError(1031), so 2,664 lvols never -# drained from in_deletion. Exit after this many consecutive failures so the -# orchestrator restarts us with a clean FDB connection (~3 min at the 3s tick). -_DB_FAILURE_RESTART_THRESHOLD = 60 - - def _log_sync_delete_failure(task, node, lvol_bdev_name, msg): """Record a failed sync delete in the cluster event log. @@ -66,45 +54,16 @@ def _log_sync_delete_failure(task, node, lvol_bdev_name, msg): def main(): logger.info("Starting Tasks runner...") - _consecutive_db_failures = 0 while True: - try: - clusters = db.get_clusters() - except Exception as e: - _consecutive_db_failures += 1 - logger.error(f"Failed to get clusters ({_consecutive_db_failures}): {e}") - if _consecutive_db_failures >= _DB_FAILURE_RESTART_THRESHOLD: - logger.error("FDB unreadable for too long; exiting for a clean restart") - sys.exit(1) - time.sleep(3) - continue - + clusters = db.get_clusters() if not clusters: - _consecutive_db_failures += 1 - logger.error(f"No clusters found! ({_consecutive_db_failures})") - if _consecutive_db_failures >= _DB_FAILURE_RESTART_THRESHOLD: - logger.error("No clusters readable for too long (FDB client likely wedged); " - "exiting for a clean restart") - sys.exit(1) + logger.error("No clusters found!") else: for cl in clusters: if cl.status == Cluster.STATUS_IN_ACTIVATION: continue - # An unhandled FDBError here (e.g. 1031 transaction timeout, - # 2026-07-16 run) killed the whole runner — count it toward - # the same wedge-detection threshold as get_clusters instead. - try: - tasks = db.get_job_tasks(cl.get_id(), reverse=False) - except Exception as e: - _consecutive_db_failures += 1 - logger.error(f"Failed to read tasks for cluster {cl.get_id()} " - f"({_consecutive_db_failures}): {e}") - if _consecutive_db_failures >= _DB_FAILURE_RESTART_THRESHOLD: - logger.error("FDB unreadable for too long; exiting for a clean restart") - sys.exit(1) - continue - _consecutive_db_failures = 0 + tasks = db.get_job_tasks(cl.get_id(), reverse=False) for task in tasks: if task.function_name == JobSchedule.FN_LVOL_SYNC_OP: if task.status != JobSchedule.STATUS_DONE: From 52b9df7a377f0abc40bb9058404817e6671e23a8 Mon Sep 17 00:00:00 2001 From: Max Schettler Date: Sat, 18 Jul 2026 09:16:52 +0200 Subject: [PATCH 02/25] Unify task retry semantics --- simplyblock_core/services/tasks_runner_backup.py | 4 ++-- simplyblock_core/services/tasks_runner_fdb_backup.py | 2 +- simplyblock_core/services/tasks_runner_jc_comp.py | 2 +- simplyblock_core/services/tasks_runner_node_add.py | 2 +- simplyblock_core/services/tasks_runner_replication_final.py | 2 +- simplyblock_core/services/tasks_runner_restart.py | 4 ++-- 6 files changed, 8 insertions(+), 8 deletions(-) diff --git a/simplyblock_core/services/tasks_runner_backup.py b/simplyblock_core/services/tasks_runner_backup.py index dfb1a5e49c..2c28c75b02 100644 --- a/simplyblock_core/services/tasks_runner_backup.py +++ b/simplyblock_core/services/tasks_runner_backup.py @@ -423,9 +423,9 @@ def process_task(task, cl): # Retry ceiling: every other task runner enforces this. Without it a task # whose step keeps failing (e.g. an RPC that crashes SPDK) loops until the - # timeout, re-triggering the failure each cycle. max_retry <= 0 means the + # timeout, re-triggering the failure each cycle. max_retry < 0 means the # task is intentionally unbounded and only the timeout applies. - if task.max_retry > 0 and task.retry >= task.max_retry: + if 0 <= task.max_retry <= task.retry: _terminate_task(task, f"max retry reached ({task.retry}/{task.max_retry})") return diff --git a/simplyblock_core/services/tasks_runner_fdb_backup.py b/simplyblock_core/services/tasks_runner_fdb_backup.py index 8fb738d42d..429e4063f7 100644 --- a/simplyblock_core/services/tasks_runner_fdb_backup.py +++ b/simplyblock_core/services/tasks_runner_fdb_backup.py @@ -21,7 +21,7 @@ def process_fdb_backup_task(task): task.write_to_db(db.kv_store) return - if task.retry >= task.max_retry: + if 0 <= task.max_retry <= task.retry: task.function_result = "max retry reached, stopping task" task.status = JobSchedule.STATUS_DONE task.write_to_db(db.kv_store) diff --git a/simplyblock_core/services/tasks_runner_jc_comp.py b/simplyblock_core/services/tasks_runner_jc_comp.py index b325b51365..40c2306def 100644 --- a/simplyblock_core/services/tasks_runner_jc_comp.py +++ b/simplyblock_core/services/tasks_runner_jc_comp.py @@ -58,7 +58,7 @@ def main(): task.write_to_db(db.kv_store) continue - if task.retry >= task.max_retry: + if 0 <= task.max_retry <= task.retry: task.function_result = "max retry reached, stopping task" task.status = JobSchedule.STATUS_DONE task.write_to_db(db.kv_store) diff --git a/simplyblock_core/services/tasks_runner_node_add.py b/simplyblock_core/services/tasks_runner_node_add.py index 224690b163..9756988651 100644 --- a/simplyblock_core/services/tasks_runner_node_add.py +++ b/simplyblock_core/services/tasks_runner_node_add.py @@ -52,7 +52,7 @@ def process_task(task, cl): task.write_to_db(db.kv_store) return False - if task.retry >= task.max_retry: + if 0 <= task.max_retry <= task.retry: task.function_result = "max retry reached" task.status = JobSchedule.STATUS_DONE task.write_to_db(db.kv_store) diff --git a/simplyblock_core/services/tasks_runner_replication_final.py b/simplyblock_core/services/tasks_runner_replication_final.py index f9b4837875..485021e4bd 100644 --- a/simplyblock_core/services/tasks_runner_replication_final.py +++ b/simplyblock_core/services/tasks_runner_replication_final.py @@ -112,7 +112,7 @@ def task_runner(task: JobSchedule): if not lvol_id: return _finalize(task, False, "missing lvol_id in task params") - if task.retry >= task.max_retry or task.canceled is True: + if (0 <= task.max_retry <= task.retry) or task.canceled is True: task.function_result = "task cancelled" if task.canceled else "max retry reached" task.status = JobSchedule.STATUS_DONE task.write_to_db(db.kv_store) diff --git a/simplyblock_core/services/tasks_runner_restart.py b/simplyblock_core/services/tasks_runner_restart.py index d90df63a74..374051ca85 100644 --- a/simplyblock_core/services/tasks_runner_restart.py +++ b/simplyblock_core/services/tasks_runner_restart.py @@ -241,7 +241,7 @@ def task_runner(task): def task_runner_device(task): device = _get_device(task) - if task.retry >= constants.TASK_EXEC_RETRY_COUNT: + if 0 <= task.max_retry <= task.retry: task.function_result = "max retry reached" task.status = JobSchedule.STATUS_DONE task.write_to_db(db.kv_store) @@ -323,7 +323,7 @@ def task_runner_node(task): _task_finish(task, "node not found") return True - if task.retry >= task.max_retry: + if 0 <= task.max_retry <= task.retry: if not _task_finish(task, "max retry reached"): # Concurrently finished/canceled — someone else owns the outcome; # the give-up side effects (OFFLINE flip + re-queue) must not run. From 4aff9042ed81c963a62ba5511a3e94c8358aa632 Mon Sep 17 00:00:00 2001 From: Max Schettler Date: Sat, 18 Jul 2026 09:50:40 +0200 Subject: [PATCH 03/25] Apply task lease consistently to all task runners This is strictly only necessary for task runners that run concurrently. It guards against incidental duplicate execution during rollouts, and prepares tasks to be parallelizable. --- simplyblock_core/services/tasks_runner_backup.py | 12 +++++++++++- .../services/tasks_runner_failed_migration.py | 3 +++ simplyblock_core/services/tasks_runner_fdb_backup.py | 5 ++++- simplyblock_core/services/tasks_runner_jc_comp.py | 3 +++ .../services/tasks_runner_new_dev_migration.py | 3 +++ .../services/tasks_runner_replication_final.py | 4 ++++ .../services/tasks_runner_sync_lvol_del.py | 6 ++++++ tests/unit/tasks/test_retry_ceiling.py | 9 +++------ 8 files changed, 37 insertions(+), 8 deletions(-) diff --git a/simplyblock_core/services/tasks_runner_backup.py b/simplyblock_core/services/tasks_runner_backup.py index 2c28c75b02..35d8240e75 100644 --- a/simplyblock_core/services/tasks_runner_backup.py +++ b/simplyblock_core/services/tasks_runner_backup.py @@ -10,7 +10,7 @@ import time from simplyblock_core import constants, db_controller, utils -from simplyblock_core.controllers import backup_events +from simplyblock_core.controllers import backup_events, tasks_controller from simplyblock_core.models.backup import Backup from simplyblock_core.models.cluster import Cluster from simplyblock_core.models.job_schedule import JobSchedule @@ -456,9 +456,19 @@ def process_task(task, cl): tasks = db.get_job_tasks(cl.get_id(), reverse=False) for task in tasks: + if task.function_name not in ( + JobSchedule.FN_BACKUP, + JobSchedule.FN_BACKUP_RESTORE, + JobSchedule.FN_BACKUP_MERGE, + ): + continue if task.status == JobSchedule.STATUS_DONE or task.canceled: continue + if not tasks_controller.claim_task(task): + logger.info(f"Backup task {task.uuid} owned by another runner host; skipping") + continue + # Re-fetch task for freshness task = db.get_task_by_id(task.uuid) process_task(task, cl) diff --git a/simplyblock_core/services/tasks_runner_failed_migration.py b/simplyblock_core/services/tasks_runner_failed_migration.py index 6ea8cf5f79..df0f075e03 100644 --- a/simplyblock_core/services/tasks_runner_failed_migration.py +++ b/simplyblock_core/services/tasks_runner_failed_migration.py @@ -158,6 +158,9 @@ def main(): if task.status != JobSchedule.STATUS_DONE: # get new task object because it could be changed from cancel task task = db.get_task_by_id(task.uuid) + if not tasks_controller.claim_task(task): + logger.info(f"Failed-migration task {task.uuid} owned by another runner host; skipping") + continue res = task_runner(task) if not res: time.sleep(3) diff --git a/simplyblock_core/services/tasks_runner_fdb_backup.py b/simplyblock_core/services/tasks_runner_fdb_backup.py index 429e4063f7..91bbb4c5a5 100644 --- a/simplyblock_core/services/tasks_runner_fdb_backup.py +++ b/simplyblock_core/services/tasks_runner_fdb_backup.py @@ -3,7 +3,7 @@ from simplyblock_core import db_controller, utils, constants -from simplyblock_core.controllers import fdb_backup_controller +from simplyblock_core.controllers import fdb_backup_controller, tasks_controller from simplyblock_core.models.job_schedule import JobSchedule from simplyblock_core.models.cluster import Cluster @@ -54,6 +54,9 @@ def process_fdb_backup_task(task): for task in tasks: if task.status != JobSchedule.STATUS_DONE: if task.function_name == JobSchedule.FN_FDB_BACKUP: + if not tasks_controller.claim_task(task): + logger.info(f"FDB-backup task {task.uuid} owned by another runner host; skipping") + continue process_fdb_backup_task(task) time.sleep(constants.TASK_EXEC_INTERVAL_SEC) diff --git a/simplyblock_core/services/tasks_runner_jc_comp.py b/simplyblock_core/services/tasks_runner_jc_comp.py index 40c2306def..22099afa9e 100644 --- a/simplyblock_core/services/tasks_runner_jc_comp.py +++ b/simplyblock_core/services/tasks_runner_jc_comp.py @@ -31,6 +31,9 @@ def main(): if task.function_name == JobSchedule.FN_JC_COMP_RESUME: if task.status != JobSchedule.STATUS_DONE: + if not tasks_controller.claim_task(task): + logger.info(f"JC-comp task {task.uuid} owned by another runner host; skipping") + continue # get new task object because it could be changed from cancel task task = db.get_task_by_id(task.uuid) diff --git a/simplyblock_core/services/tasks_runner_new_dev_migration.py b/simplyblock_core/services/tasks_runner_new_dev_migration.py index 76b611e59e..a74af6b0ce 100644 --- a/simplyblock_core/services/tasks_runner_new_dev_migration.py +++ b/simplyblock_core/services/tasks_runner_new_dev_migration.py @@ -186,6 +186,9 @@ def main(): if task.status != JobSchedule.STATUS_DONE: # get new task object because it could be changed from cancel task task = db.get_task_by_id(task.uuid) + if not tasks_controller.claim_task(task): + logger.info(f"New-device migration task {task.uuid} owned by another runner host; skipping") + continue res = task_runner(task) if not res: time.sleep(2) diff --git a/simplyblock_core/services/tasks_runner_replication_final.py b/simplyblock_core/services/tasks_runner_replication_final.py index 485021e4bd..66df139c7f 100644 --- a/simplyblock_core/services/tasks_runner_replication_final.py +++ b/simplyblock_core/services/tasks_runner_replication_final.py @@ -25,6 +25,7 @@ from datetime import datetime from simplyblock_core import constants, db_controller, utils +from simplyblock_core.controllers import tasks_controller from simplyblock_core.models.job_schedule import JobSchedule from simplyblock_core.models.lvol_model import LVol, LVolReplication from simplyblock_core.models.snapshot import SnapShot @@ -287,6 +288,9 @@ def main(): if task.status == JobSchedule.STATUS_DONE: continue task = db.get_task_by_id(task.uuid) + if not tasks_controller.claim_task(task): + logger.info(f"Replication-final task {task.uuid} owned by another runner host; skipping") + continue try: res = task_runner(task) except Exception as e: diff --git a/simplyblock_core/services/tasks_runner_sync_lvol_del.py b/simplyblock_core/services/tasks_runner_sync_lvol_del.py index 99b16cd197..1068134c74 100644 --- a/simplyblock_core/services/tasks_runner_sync_lvol_del.py +++ b/simplyblock_core/services/tasks_runner_sync_lvol_del.py @@ -67,6 +67,9 @@ def main(): for task in tasks: if task.function_name == JobSchedule.FN_LVOL_SYNC_OP: if task.status != JobSchedule.STATUS_DONE: + if not tasks_controller.claim_task(task): + logger.info(f"LVol sync task {task.uuid} owned by another runner host; skipping") + continue # Re-read (it may have been canceled concurrently). task = db.get_task_by_id(task.uuid) if task.status == JobSchedule.STATUS_DONE: @@ -79,6 +82,9 @@ def main(): if task.function_name == JobSchedule.FN_LVOL_SYNC_DEL: if task.status != JobSchedule.STATUS_DONE: + if not tasks_controller.claim_task(task): + logger.info(f"LVol sync task {task.uuid} owned by another runner host; skipping") + continue # get new task object because it could be changed from cancel task task = db.get_task_by_id(task.uuid) diff --git a/tests/unit/tasks/test_retry_ceiling.py b/tests/unit/tasks/test_retry_ceiling.py index 0b4a429b70..5744b0c23b 100644 --- a/tests/unit/tasks/test_retry_ceiling.py +++ b/tests/unit/tasks/test_retry_ceiling.py @@ -287,6 +287,9 @@ def _wire_base(runner, monkeypatch, task): monkeypatch.setattr(runner, "db", db) monkeypatch.setattr(JobSchedule, "write_to_db", MagicMock()) _patch_clock(monkeypatch, runner, task) + # Every runner claims the lease before running a task; grant it so the loop + # reaches the work under test instead of skipping the task as foreign-owned. + monkeypatch.setattr(runner.tasks_controller, "claim_task", lambda *a, **k: True) return db, cluster, node @@ -302,8 +305,6 @@ def _wire_base(runner, monkeypatch, task): def _spec_cluster_expand(runner, monkeypatch): task = _make_task(JobSchedule.FN_CLUSTER_EXPAND, new_node_id="new-1") _wire_base(runner, monkeypatch, task) - monkeypatch.setattr(runner.tasks_controller, "claim_task", - lambda *a, **k: True) # The actual expansion work fails every cycle. monkeypatch.setattr( runner, "integrate_new_node_into_cluster", @@ -316,8 +317,6 @@ def _spec_node_add(runner, monkeypatch): _wire_base(runner, monkeypatch, task) monkeypatch.setattr(runner, "ThreadPoolExecutor", _InlineExecutor) monkeypatch.setattr(runner, "_inflight", set()) - monkeypatch.setattr(runner.tasks_controller, "claim_task", - lambda *a, **k: True) # add_node fails (returns falsy) every cycle. monkeypatch.setattr(runner.storage_node_ops, "add_node", MagicMock(return_value=False)) @@ -355,8 +354,6 @@ def _spec_restart(runner, monkeypatch): monkeypatch.setattr(runner, "_restart_next_attempt", {}) monkeypatch.setattr(runner, "_restart_inflight", {}) monkeypatch.setattr(runner, "_node_inflight", {}) - monkeypatch.setattr(runner.tasks_controller, "claim_task", - lambda *a, **k: True) monkeypatch.setattr(runner.tasks_controller, "is_auto_restart_paused", lambda *a, **k: False) monkeypatch.setattr(runner.tasks_controller, "add_node_to_auto_restart", From c49b88b43c7f09650dbdbd5d8a417adbe90d6b95 Mon Sep 17 00:00:00 2001 From: Max Schettler Date: Sat, 18 Jul 2026 10:15:30 +0200 Subject: [PATCH 04/25] Reclassify backup policy evaluation as service The executor does not drive execution of a specific task but simply periodically evaluates backup policies. This matches the pattern that existing services have, so the executor is reclassified. --- scripts/collect_logs.py | 3012 ++++++++--------- simplyblock_core/scripts/collect_logs.py | 2 +- .../scripts/docker-compose-swarm.yml | 4 +- ...ackup_merge.py => backup_merge_service.py} | 8 +- 4 files changed, 1510 insertions(+), 1516 deletions(-) rename simplyblock_core/services/{tasks_runner_backup_merge.py => backup_merge_service.py} (83%) diff --git a/scripts/collect_logs.py b/scripts/collect_logs.py index 84d7de41ad..a88c99690e 100755 --- a/scripts/collect_logs.py +++ b/scripts/collect_logs.py @@ -1,1506 +1,1506 @@ -#!/usr/bin/env python3 -""" -Simplyblock Log Collector -========================= -Collects container logs from Graylog (or directly from OpenSearch) for a -specified time window, organises them by storage node and control-plane -service, and packages everything into a compressed tarball. - -The script must be run on a management node or inside an admin pod where -the `sbctl` CLI is available and has full admin access. - -Usage ------ - collect_logs.py [options] - - start_time ISO-8601 datetime, UTC assumed when no timezone given. - Accepted formats: "2024-01-15T10:00:00" - "2024-01-15 10:00:00" - "2024-01-15T10:00:00+00:00" - - duration_minutes Number of minutes to collect from start_time. - -Options -------- - --output-dir DIR Write the tarball here (default: current directory). - --mode MODE Deployment mode: "docker" (default) or "kubernetes". - Selects the set of control-plane service names and - adjusts which log sources are queried. - --use-opensearch Query OpenSearch scroll API directly instead of the - Graylog search REST API. Useful when Graylog is - unavailable or when the result set is very large. - --cluster-id UUID Force a specific cluster UUID (default: first cluster). - --mgmt-ip IP Override management-node IP for Graylog / OpenSearch. - -Examples --------- - collect_logs.py "2024-01-15T10:00:00" 60 - collect_logs.py "2024-01-15 10:00:00" 30 --output-dir /tmp/logs - collect_logs.py "2024-01-15T10:00:00" 120 --use-opensearch - collect_logs.py "2024-01-15T10:00:00" 60 --mode kubernetes -""" - -import argparse -import json -import os -import shutil -import subprocess -import sys -import tarfile -import tempfile -from datetime import datetime, timezone, timedelta -from pathlib import Path -from typing import Any - -try: - import requests -except ImportError: - print( - "ERROR: the 'requests' library is required.\n" - " Install it with: pip3 install requests", - file=sys.stderr, - ) - sys.exit(1) - -# --------------------------------------------------------------------------- -# Constants -# --------------------------------------------------------------------------- - -# Maximum records per single Graylog search page. -PAGE_SIZE = 1000 - -# OpenSearch max_result_window is set to 100 000 during cluster initialisation -# (see simplyblock_core/cluster_ops.py :: _set_max_result_window). -# Requests that would exceed this threshold are split into time-based chunks. -MAX_RESULT_WINDOW = 100_000 - -# Docker Swarm service names that run on the management / control-plane node. -CONTROL_PLANE_SERVICES_DOCKER = [ - "WebAppAPI", - "WebAppAPI2", - "WebAppAPI3", - "WebAppAPI4", - "WebAppAPI5", - "fdb-server", - "fdb-backup-agent", - "StorageNodeMonitor", - "MgmtNodeMonitor", - "LVolStatsCollector", - "MainDistrEventCollector", - "CapacityAndStatsCollector", - "CapacityMonitor", - "HealthCheck", - "DeviceMonitor", - "LVolMonitor", - "SnapshotMonitor", - "TasksRunnerRestart", - "TasksRunnerMigration", - "TasksRunnerLVolMigration", - "TasksRunnerFailedMigration", - "TasksRunnerClusterStatus", - "TasksRunnerNewDeviceMigration", - "TasksNodeAddRunner", - "TasksRunnerClusterExpand", - "TasksRunnerPortAllow", - "TasksRunnerJCCompResume", - "TasksRunnerLVolSyncDelete", - "TasksRunnerBackup", - "TasksRunnerBackupMerge", - # Async cross-cluster replication: these run on the CP but were missing from - # this list, so a replication incident collected no replication logs at all. - "SnapshotReplication", - "TasksRunnerReplicationFinal", - "TasksRunnerBatchMigration", - "TasksNodeRemovalRunner", - "HAProxy", -] - -CONTROL_PLANE_SERVICES_KUBERNETES = [ - "simplyblock-control", - "webappapi", - "storage-node-monitor", - "mgmt-node-monitor", - "lvol-stats-collector", - "main-distr-event-collector", - "capacity-and-stats-collector", - "capacity-monitor", - "health-check", - "device-monitor", - "lvol-monitor", - "snapshot-monitor", - "tasks-node-add-runner", - "tasks-runner-restart", - "tasks-runner-migration", - "tasks-runner-failed-migration", - "tasks-runner-cluster-status", - "tasks-runner-new-device-migration", - "tasks-runner-port-allow", - "tasks-runner-jc-comp-resume", - "tasks-runner-sync-lvol-del", - "tasks-runner-backup", - "tasks-runner-backup-merge", - "tasks-runner-snapshot-replication", - "tasks-runner-replication-final", - "tasks-runner-batch-migration", -] - -# --------------------------------------------------------------------------- -# sbctl helpers -# --------------------------------------------------------------------------- - - -def _sbctl_bin(): - """Absolute path to the sbctl binary. - - The script is normally run under sudo, whose secure_path does not include - /usr/local/bin, so a bare "sbctl" raises FileNotFoundError and the collector - exits before gathering anything. - """ - found = shutil.which("sbctl") - if found: - return found - for candidate in ("/usr/local/bin/sbctl", "/usr/bin/sbctl"): - if os.path.isfile(candidate) and os.access(candidate, os.X_OK): - return candidate - return "sbctl" - - -SBCTL_BIN = _sbctl_bin() - - -def _collect_docker_service_logs(cp_dir, tail=40000): - """Fallback: pull control-plane logs straight from Docker Swarm. - - Used when Graylog/OpenSearch returns nothing. `docker service logs` - aggregates ALL tasks of a service, including ones that were replaced during - the window — `docker logs ` only has the currently running task, - so a service recreated mid-incident loses its earlier output entirely. - - Best-effort: this only works on a docker-mode management node, and any - failure is reported rather than raised so the rest of the bundle survives. - """ - print(" -> falling back to `docker service logs` on this node") - r = _run(["docker", "service", "ls", "--format", "{{.Name}}"], timeout=60) - if r is None or r.returncode != 0: - print(" !! cannot list docker services; no control-plane logs in this bundle") - return - services = [s.strip() for s in r.stdout.splitlines() if s.strip()] - if not services: - print(" !! no docker services found") - return - - out_dir = cp_dir / "docker_service_logs" - out_dir.mkdir(exist_ok=True) - total = 0 - for svc in services: - r = _run(["docker", "service", "logs", "--timestamps", "--no-task-ids", - "--tail", str(tail), svc], timeout=600) - if r is None: - print(f" {svc:<42} {'timed out':>8}") - continue - body = (r.stdout or "") + (r.stderr or "") - (out_dir / f"{svc}.log").write_text(body, encoding="utf-8", errors="replace") - lines = body.count("\n") - total += lines - print(f" {svc:<42} {lines:>8,} lines") - print(f" {'Docker service logs total':<42} {total:>8,} lines") - - -def _run(cmd, timeout=30): - """Run *cmd* list; return CompletedProcess or None on failure.""" - try: - return subprocess.run(cmd, capture_output=True, text=True, timeout=timeout) - except FileNotFoundError: - print(f"ERROR: command not found: {cmd[0]}", file=sys.stderr) - sys.exit(1) - except subprocess.TimeoutExpired: - print(f"ERROR: command timed out: {' '.join(cmd)}", file=sys.stderr) - return None - - -def sbctl_json(*args): - """ - Run ``sbctl --json`` and return the parsed JSON (list or dict). - Returns None and prints an error on failure. - """ - cmd = [SBCTL_BIN] + list(args) + ["--json"] - r = _run(cmd) - if r is None or r.returncode != 0: - if r: - print(f"ERROR: {' '.join(cmd)}\n stderr: {r.stderr.strip()}", file=sys.stderr) - return None - try: - return json.loads(r.stdout) - except json.JSONDecodeError: - print( - f"ERROR: could not parse JSON from: {' '.join(cmd)}\n" - f" output: {r.stdout[:400]}", - file=sys.stderr, - ) - return None - - -def sbctl_raw(*args): - """ - Run ``sbctl `` (no --json) and return stripped stdout text. - Returns None on failure. - """ - r = _run([SBCTL_BIN] + list(args)) - if r is None or r.returncode != 0: - if r: - print( - f"ERROR: sbctl {' '.join(args)}\n stderr: {r.stderr.strip()}", - file=sys.stderr, - ) - return None - return r.stdout.strip() - - -# --------------------------------------------------------------------------- -# Log-line formatter -# --------------------------------------------------------------------------- - - -def _fmt(msg: dict) -> str: - """Render a Graylog / OpenSearch message dict as a single log line.""" - ts = msg.get("timestamp", "") - src = msg.get("source", "") - cname = msg.get("container_name", "") - lvl = msg.get("level", "") - text = str(msg.get("message", "")).replace("\n", "\\n") - return f"{ts} src={src} ctr={cname} lvl={lvl} {text}" - - -# --------------------------------------------------------------------------- -# Graylog REST API helpers -# --------------------------------------------------------------------------- - -def _gl_escape(value: str) -> str: - """ - Escape Lucene special characters in a Graylog field query term. - Hyphens are NOT escaped — they are only special in range expressions - and cause HTTP 400 when escaped in the Graylog REST API. - """ - return value.replace(".", "\\.") - - -def _gl_search_page(session, search_url, query, from_iso, to_iso, limit, offset): - """ - Fetch one page of results from the Graylog absolute-search endpoint. - Returns (messages_list, total_results) or (None, 0) on error. - """ - params = { - "query": query, - "from": from_iso, - "to": to_iso, - "limit": limit, - "offset": offset, - "sort": "timestamp:asc", - "fields": "timestamp,source,container_name,level,message", - } - try: - resp = session.get(search_url, params=params, timeout=90, - headers={"Accept": "application/json"}) - resp.raise_for_status() - except requests.RequestException as exc: - print(f" WARN: Graylog page request failed (offset={offset}): {exc}", file=sys.stderr) - return None, 0 - - if not resp.text.strip(): - print(f" WARN: Graylog returned empty response (offset={offset}, status={resp.status_code})", file=sys.stderr) - return None, 0 - try: - data = resp.json() - except requests.exceptions.JSONDecodeError as exc: - print(f" WARN: Graylog response is not valid JSON (offset={offset}): {exc}", file=sys.stderr) - return None, 0 - return data.get("messages", []), data.get("total_results", 0) - - -def _gl_write_window(session, search_url, query, from_iso, to_iso, fh): - """ - Paginate through a single time window and write ALL lines to *fh*. - - The window MUST have total <= MAX_RESULT_WINDOW. Caller is responsible - for checking the count first (via ``_gl_probe_total``) and bisecting - the window when it exceeds the limit. - - Returns number of lines written. - """ - written = 0 - offset = 0 - - while True: - msgs, _ = _gl_search_page( - session, search_url, query, from_iso, to_iso, PAGE_SIZE, offset - ) - if msgs is None: - break - if not msgs: - break - for m in msgs: - fh.write(_fmt(m.get("message", {})) + "\n") - written += 1 - offset += len(msgs) - if len(msgs) < PAGE_SIZE: - break - if offset >= MAX_RESULT_WINDOW: - break - - return written - - -def _gl_probe_total(session, search_url, query, from_iso, to_iso): - """Return total number of matching entries for a window, or -1 on error.""" - msgs, total = _gl_search_page(session, search_url, query, from_iso, to_iso, 1, 0) - if msgs is None: - return -1 - return total - - -# Minimum bisection window size in seconds. Windows smaller than this are -# fetched best-effort (capped at MAX_RESULT_WINDOW). -MIN_BISECT_SEC = 1 - - -def _gl_fetch_recursive(session, search_url, query, from_dt, to_dt, fh, - depth=0): - """ - Recursively bisect the time window until entries fit within - MAX_RESULT_WINDOW, then paginate and write. - - Returns ``(lines_written, truncated_count)`` where *truncated_count* - is the number of leaf windows that exceeded the limit even at the - minimum window size (data loss). - """ - from_iso = from_dt.strftime("%Y-%m-%dT%H:%M:%S.000Z") - to_iso = to_dt.strftime("%Y-%m-%dT%H:%M:%S.000Z") - window_sec = (to_dt - from_dt).total_seconds() - indent = " " + " " * min(depth, 6) - - total = _gl_probe_total(session, search_url, query, from_iso, to_iso) - if total <= 0: - return 0, 0 - - # Window fits — fetch everything - if total <= MAX_RESULT_WINDOW: - written = _gl_write_window(session, search_url, query, - from_iso, to_iso, fh) - return written, 0 - - # Window too big — can we bisect further? - if window_sec <= MIN_BISECT_SEC: - # Smallest window reached — best-effort fetch (capped at 100k) - print(f"{indent}WARN: {window_sec:.0f}s window at {from_iso} has " - f"{total} entries (>{MAX_RESULT_WINDOW}), capturing first " - f"{MAX_RESULT_WINDOW} (best effort)", file=sys.stderr) - written = _gl_write_window(session, search_url, query, - from_iso, to_iso, fh) - return written, 1 - - # Bisect - mid_dt = from_dt + (to_dt - from_dt) / 2 - print(f"{indent}NOTE: {from_iso}..{to_iso} ({window_sec:.0f}s) has " - f"{total} entries, bisecting") - - w1, t1 = _gl_fetch_recursive(session, search_url, query, - from_dt, mid_dt, fh, depth + 1) - w2, t2 = _gl_fetch_recursive(session, search_url, query, - mid_dt, to_dt, fh, depth + 1) - return w1 + w2, t1 + t2 - - -def graylog_fetch_all(session, base_url, query, from_iso, to_iso, out_path): - """ - Download all log messages matching *query* within [from_iso, to_iso]. - - Uses recursive binary bisection: probe the window count first, and if - it exceeds MAX_RESULT_WINDOW, split the window in half and recurse. - Keeps halving down to MIN_BISECT_SEC (1 second). Only writes data at - leaf windows that fit within the limit (no duplicate writes). - - Writes one text line per message to *out_path*. - Returns ``(lines_written, truncated_count)`` where *truncated_count* - is the number of leaf windows that still exceeded the limit at the - minimum window size. - """ - search_url = f"{base_url}/search/universal/absolute" - - # Quick probe - total = _gl_probe_total(session, search_url, query, from_iso, to_iso) - if total < 0: - Path(out_path).touch() - return 0, 0 - if total == 0: - Path(out_path).touch() - print(" total entries: 0") - return 0, 0 - - print(f" total entries: {total}") - - from_dt = datetime.fromisoformat(from_iso.replace("Z", "+00:00")) - to_dt = datetime.fromisoformat(to_iso.replace("Z", "+00:00")) - - with open(out_path, "w") as fh: - written, truncated = _gl_fetch_recursive( - session, search_url, query, from_dt, to_dt, fh, - ) - - return written, truncated - - -# --------------------------------------------------------------------------- -# OpenSearch scroll API helpers (--use-opensearch) -# --------------------------------------------------------------------------- - - -def _os_get_index(session, os_url): - """ - Discover the graylog indices present in OpenSearch and return them as a - comma-separated string suitable for use in a URL path segment. - - Using _cat/indices avoids embedding a '*' wildcard in the URL, which - HAProxy may reject (400). Falls back to '_all' if discovery fails. - """ - try: - r = session.get(f"{os_url}/_cat/indices?h=index&format=json", timeout=10) - r.raise_for_status() - indices = sorted( - i["index"] - for i in r.json() - if i["index"].startswith("graylog") and not i["index"].startswith(".") - ) - if indices: - return ",".join(indices) - except Exception as exc: - print(f" WARN: could not discover OpenSearch indices ({exc}); using _all", file=sys.stderr) - return "_all" - - -def _os_probe(session, os_url, index, from_ms, to_ms): - """ - Probe the index to discover: - - The actual timestamp field name (e.g. 'timestamp' vs '@timestamp') - - The actual container-name field name - - How many documents exist in the requested time window (any container) - - A sample document so we can see real field values - - Returns a dict with keys: ts_field, cname_field, window_count, sample_doc - """ - result = {"ts_field": "timestamp", "cname_field": "container_name", - "window_count": 0, "sample_doc": None} - - # --- sample document (no time filter) --- - try: - r = session.post( - f"{os_url}/{index}/_search", - json={"size": 1, "query": {"match_all": {}}}, - timeout=10, - ) - if r.ok: - hits = r.json().get("hits", {}).get("hits", []) - if hits: - src = hits[0].get("_source", {}) - result["sample_doc"] = src - # Detect timestamp field - if "@timestamp" in src: - result["ts_field"] = "@timestamp" - # Detect container-name field (various naming conventions) - for candidate in ("kubernetes_container_name", "container_name", - "container_id", "containerName", - "_container_name", "docker_container_name"): - if candidate in src: - result["cname_field"] = candidate - break - except Exception as exc: - print(f" WARN: probe (sample doc) failed: {exc}", file=sys.stderr) - - # --- count within the requested time window --- - ts = result["ts_field"] - try: - r = session.post( - f"{os_url}/{index}/_count", - json={"query": {"range": {ts: {"gte": from_ms, "lte": to_ms, - "format": "epoch_millis"}}}}, - timeout=10, - ) - if r.ok: - result["window_count"] = r.json().get("count", 0) - except Exception as exc: - print(f" WARN: probe (window count) failed: {exc}", file=sys.stderr) - - return result - - -def _os_sample_container_names(session, os_url, index, from_ms, to_ms, ts_field, cname_field, n=30): - """ - Return up to *n* distinct container_name values within the time window - using a terms aggregation. Used by --diagnose. - """ - body = { - "size": 0, - "query": {"range": {ts_field: {"gte": from_ms, "lte": to_ms, - "format": "epoch_millis"}}}, - "aggs": { - "names": { - "terms": { - "field": f"{cname_field}.keyword", - "size": n, - } - } - }, - } - try: - r = session.post(f"{os_url}/{index}/_search", json=body, timeout=15) - if r.ok: - buckets = r.json().get("aggregations", {}).get("names", {}).get("buckets", []) - return [(b["key"], b["doc_count"]) for b in buckets] - except Exception: - pass - return [] - - -def opensearch_diagnose(session, os_url, from_iso, to_iso): - """ - Print a detailed diagnostic report about what is in OpenSearch. - Called when --diagnose is passed. - """ - print("\n" + "=" * 64) - print(" OpenSearch Diagnostic Report") - print("=" * 64) - - from_ms = int(datetime.fromisoformat(from_iso.replace("Z", "+00:00")).timestamp() * 1000) - to_ms = int(datetime.fromisoformat(to_iso.replace("Z", "+00:00")).timestamp() * 1000) - - # 1. List all indices - print("\n[D1] All indices:") - try: - r = session.get(f"{os_url}/_cat/indices?h=index,docs.count,store.size&format=json", - timeout=10) - r.raise_for_status() - for idx in sorted(r.json(), key=lambda x: x["index"]): - print(f" {idx['index']:<45} docs={idx.get('docs.count','?'):>10} " - f"size={idx.get('store.size','?')}") - except Exception as exc: - print(f" ERROR: {exc}") - - index = _os_get_index(session, os_url) - print(f"\n → Using index(es): {index}") - - # 2. Probe - probe = _os_probe(session, os_url, index, from_ms, to_ms) - print("\n[D2] Detected field names:") - print(f" timestamp field : {probe['ts_field']}") - print(f" container_name field: {probe['cname_field']}") - print(f"\n[D3] Documents in requested time window: {probe['window_count']}") - - # 3. Sample document - if probe["sample_doc"]: - print("\n[D4] Sample document fields and values:") - for k, v in sorted(probe["sample_doc"].items()): - v_str = str(v)[:120] - print(f" {k:<35} = {v_str}") - else: - print("\n[D4] No sample document found (index may be empty).") - - # 4. Container names in window - print("\n[D5] Distinct container_name values in time window (up to 30):") - names = _os_sample_container_names(session, os_url, index, - from_ms, to_ms, - probe["ts_field"], probe["cname_field"]) - if names: - for name, count in names: - print(f" {name:<60} {count:>8} docs") - else: - print(" (none found – aggregation on .keyword sub-field may have failed)") - print(" Trying match_all sample …") - try: - r = session.post( - f"{os_url}/{index}/_search", - json={"size": 5, "query": {"match_all": {}}, - "_source": [probe["cname_field"]]}, - timeout=10, - ) - if r.ok: - for h in r.json().get("hits", {}).get("hits", []): - print(f" {h.get('_source', {}).get(probe['cname_field'], '???')}") - except Exception: - pass - - print("\n" + "=" * 64) - - -def opensearch_fetch_all(session, os_url, container_name, source, from_iso, to_iso, out_path, - probe_cache=None, pod_name=None): - """ - Fetch logs directly from OpenSearch using the scroll API. - - Discovers the actual timestamp and container-name field names via a - one-time probe (cached in *probe_cache* dict across calls). - Uses query_string wildcards for container matching so Docker Swarm - names like 'simplyblock_WebAppAPI.1.' are matched by just - passing 'WebAppAPI'. - Returns number of lines written. - """ - # Graylog's OpenSearch index maps the timestamp field with format - # "uuuu-MM-dd HH:mm:ss.SSS" (space separator, no timezone suffix). - # epoch_millis is accepted regardless of the field's stored date format. - from_ms = int(datetime.fromisoformat(from_iso.replace("Z", "+00:00")).timestamp() * 1000) - to_ms = int(datetime.fromisoformat(to_iso.replace("Z", "+00:00")).timestamp() * 1000) - - # One-time index discovery + probe (cached) - if probe_cache is None: - probe_cache = {} - if "index" not in probe_cache: - probe_cache["index"] = _os_get_index(session, os_url) - probe_cache["probe"] = _os_probe(session, os_url, probe_cache["index"], from_ms, to_ms) - p = probe_cache["probe"] - print(f" [OpenSearch] index={probe_cache['index']} " - f"ts_field={p['ts_field']} cname_field={p['cname_field']} " - f"docs_in_window={p['window_count']}") - if p["window_count"] == 0: - print(" WARN: no documents in the requested time window – " - "check the start_time / duration, or run with --diagnose", - file=sys.stderr) - - index = probe_cache["index"] - probe = probe_cache["probe"] - ts_f = probe["ts_field"] - cname_f = probe["cname_field"] - - # Build query - # Use query_string wildcards so partial names work: - # "WebAppAPI" matches "simplyblock_WebAppAPI.1.abc123" - # "spdk_8080" matches "/spdk_8080" - must_clauses: list[Any] = [ - {"range": {ts_f: {"gte": from_ms, "lte": to_ms, "format": "epoch_millis"}}}, - ] - if container_name: - esc = container_name.replace("/", "\\/").replace(":", "\\:") - must_clauses.append({ - "query_string": { - "default_field": cname_f, - "query": f"*{esc}*", - "analyze_wildcard": True, - } - }) - if pod_name: - esc_pod = pod_name.replace("/", "\\/").replace(":", "\\:") - must_clauses.append({ - "query_string": { - "default_field": "kubernetes_pod_name", - "query": f"*{esc_pod}*", - "analyze_wildcard": True, - } - }) - if source: - # source may be a single string or a list of candidate values - # (e.g. multiple hostname formats for the same node). - # When it is a list we OR them so any matching format succeeds. - candidates = source if isinstance(source, (list, tuple)) else [source] - if len(candidates) == 1: - must_clauses.append({ - "query_string": { - "default_field": "source", - "query": f'"{candidates[0]}"', - } - }) - else: - must_clauses.append({ - "bool": { - "should": [ - {"query_string": {"default_field": "source", - "query": f'"{c}"'}} - for c in candidates - ], - "minimum_should_match": 1, - } - }) - - body = { - "query": {"bool": {"must": must_clauses}}, - "sort": [{ts_f: {"order": "asc"}}], - "size": PAGE_SIZE, - "_source": [ts_f, "source", cname_f, "level", "message"], - } - - init_url = f"{os_url}/{index}/_search?scroll=2m" - written = 0 - - try: - r = session.post(init_url, json=body, timeout=60) - if not r.ok: - print( - f" WARN: OpenSearch initial scroll failed: {r.status_code} {r.reason}" - f"\n body: {r.text[:400]}", - file=sys.stderr, - ) - Path(out_path).touch() - return 0 - except requests.RequestException as exc: - print(f" WARN: OpenSearch initial scroll failed: {exc}", file=sys.stderr) - Path(out_path).touch() - return 0 - - data = r.json() - scroll_id = data.get("_scroll_id") - hits = data.get("hits", {}).get("hits", []) - total = data.get("hits", {}).get("total", {}) - total = total.get("value", total) if isinstance(total, dict) else int(total or 0) - print(f" total entries: {total}") - - with open(out_path, "w") as fh: - while hits: - for h in hits: - src = h.get("_source", {}) - # normalise field names to what _fmt expects - if ts_f != "timestamp": - src["timestamp"] = src.get(ts_f, "") - if cname_f != "container_name": - src["container_name"] = src.get(cname_f, "") - fh.write(_fmt(src) + "\n") - written += 1 - if len(hits) < PAGE_SIZE or not scroll_id: - break - try: - sc_r = session.post( - f"{os_url}/_search/scroll", - json={"scroll": "2m", "scroll_id": scroll_id}, - timeout=60, - ) - sc_r.raise_for_status() - sc_data = sc_r.json() - scroll_id = sc_data.get("_scroll_id", scroll_id) - hits = sc_data.get("hits", {}).get("hits", []) - except requests.RequestException as exc: - print(f" WARN: scroll continuation failed: {exc}", file=sys.stderr) - break - - # Release scroll context - if scroll_id: - try: - session.delete( - f"{os_url}/_search/scroll", - json={"scroll_id": scroll_id}, - timeout=10, - ) - except Exception: - pass - - return written - - -# --------------------------------------------------------------------------- -# Dispatch helper -# --------------------------------------------------------------------------- - - -def fetch( - *, - gl_session, - os_session, - graylog_base, - opensearch_base, - use_opensearch, - gl_query, - os_container, - os_source, - from_iso, - to_iso, - out_path, - probe_cache, - os_pod_name=None, -): - """Route to Graylog or OpenSearch depending on *use_opensearch*. - - When using Graylog and the recursive bisection still has truncated - leaf windows, automatically falls back to OpenSearch scroll API - (which has no offset limit) for the entire service. - """ - if use_opensearch: - return opensearch_fetch_all( - os_session, opensearch_base, - os_container, os_source, - from_iso, to_iso, str(out_path), - probe_cache=probe_cache, - pod_name=os_pod_name, - ) - - written, truncated = graylog_fetch_all( - gl_session, graylog_base, - gl_query, from_iso, to_iso, str(out_path), - ) - - if truncated and opensearch_base: - print(f" NOTE: Graylog had {truncated} truncated window(s), " - f"retrying via OpenSearch scroll API for complete data") - try: - os_written = opensearch_fetch_all( - os_session, opensearch_base, - os_container, os_source, - from_iso, to_iso, str(out_path), - probe_cache=probe_cache, - pod_name=os_pod_name, - ) - if os_written > 0: - return os_written - print(f" WARN: OpenSearch fallback returned 0 lines, " - f"keeping Graylog result ({written} lines)") - except Exception as exc: - print(f" WARN: OpenSearch fallback failed ({exc}), " - f"keeping Graylog result ({written} lines)", - file=sys.stderr) - - return written - - -# --------------------------------------------------------------------------- -# kubectl pod-log helpers -# --------------------------------------------------------------------------- - - -def _kubectl(*args, timeout=60) -> str: - """Run kubectl with the given args and return stdout. Returns '' on failure.""" - try: - r = subprocess.run( - ["kubectl"] + list(args), - capture_output=True, text=True, timeout=timeout, - ) - return r.stdout - except Exception as exc: - print(f" WARN: kubectl {' '.join(args[:4])} … failed: {exc}", file=sys.stderr) - return "" - - -def _kubectl_list_pods(namespace: str, prefix: str) -> list[str]: - """Return pod names in *namespace* whose name starts with *prefix*.""" - out = _kubectl("get", "pods", "-n", namespace, - "--no-headers", "-o", "custom-columns=:metadata.name") - return [p for p in out.splitlines() if p.startswith(prefix)] - - -def _kubectl_containers(namespace: str, pod: str) -> list[str]: - """Return init + regular container names for *pod*.""" - out = _kubectl( - "get", "pod", pod, "-n", namespace, - "-o", - "jsonpath={range .spec.initContainers[*]}{.name}{'\\n'}{end}" - "{range .spec.containers[*]}{.name}{'\\n'}{end}", - ) - return [c for c in out.splitlines() if c] - - -def collect_k8s_pod_logs(namespace: str, pod: str, out_dir: Path, - from_iso: str, to_iso: str) -> None: - """ - Write current + previous logs for every container in *pod* to *out_dir*. - Files are named _.log - """ - containers = _kubectl_containers(namespace, pod) - for container in containers: - log_file = out_dir / f"{pod}_{container}.log" - print(f" {pod} / {container}") - with open(log_file, "w") as fh: - fh.write(f"=== Pod: {pod} | Container: {container} | Namespace: {namespace} ===\n") - fh.write(f"=== From: {from_iso} | Until: {to_iso} ===\n\n") - - fh.write("--- current logs ---\n") - out = _kubectl("logs", pod, "-c", container, "-n", namespace, - "--timestamps", f"--since-time={from_iso}", timeout=120) - # Trim lines beyond to_iso - for line in out.splitlines(): - if line[:26] > to_iso[:26]: - break - fh.write(line + "\n") - - fh.write("\n--- previous (last crash) logs ---\n") - prev = _kubectl("logs", pod, "-c", container, "-n", namespace, - "--timestamps", "--previous", timeout=60) - fh.write(prev if prev.strip() else "(no previous logs)\n") - - -def collect_k8s_csi_dmesg(namespace: str, pod: str, out_dir: Path, - from_iso: str, to_iso: str) -> None: - """ - Collect dmesg from the csi-node container of a CSI pod, - filtered to the requested time window using the kernel boot epoch. - """ - from_epoch = int(datetime.fromisoformat(from_iso.replace("Z", "+00:00")).timestamp()) - to_epoch = int(datetime.fromisoformat(to_iso.replace("Z", "+00:00")).timestamp()) - - log_file = out_dir / f"{pod}_csi-node_dmesg.log" - print(f" {pod} / csi-node (dmesg)") - - # Derive boot epoch from /proc/uptime inside the container - uptime_out = _kubectl("exec", pod, "-c", "csi-node", "-n", namespace, - "--", "cat", "/proc/uptime", timeout=10) - try: - boot_epoch = int(datetime.now(timezone.utc).timestamp()) - int(float(uptime_out.split()[0])) - except Exception: - boot_epoch = 0 - - # Prefer human-readable reltime; fall back to monotonic seconds - dmesg_out = _kubectl("exec", pod, "-c", "csi-node", "-n", namespace, - "--", "dmesg", "--kernel", "--time-format=reltime", - "--nopager", timeout=30) - if not dmesg_out.strip(): - dmesg_out = _kubectl("exec", pod, "-c", "csi-node", "-n", namespace, - "--", "dmesg", "--kernel", "--nopager", timeout=30) - # Filter by monotonic timestamp - filtered = [] - import re - for line in dmesg_out.splitlines(): - m = re.match(r'^\[\s*([0-9]+\.[0-9]+)\]', line) - if m: - wall = boot_epoch + int(float(m.group(1))) - if wall < from_epoch: - continue - if wall > to_epoch: - break - filtered.append(line) - dmesg_out = "\n".join(filtered) - - with open(log_file, "w") as fh: - fh.write(f"=== Pod: {pod} | Container: csi-node | dmesg ===\n") - fh.write(f"=== From: {from_iso} | Until: {to_iso} ===\n\n") - fh.write(dmesg_out or "(no dmesg output)\n") - - -# --------------------------------------------------------------------------- -# Main -# --------------------------------------------------------------------------- - - -def main(): - parser = argparse.ArgumentParser( - prog="collect_logs.py", - description="Collect simplyblock container logs for a given time window.", - formatter_class=argparse.RawDescriptionHelpFormatter, - epilog=( - "Examples:\n" - ' collect_logs.py "2024-01-15T10:00:00" 60\n' - ' collect_logs.py "2024-01-15 10:00:00" 30 --output-dir /tmp/logs\n' - ' collect_logs.py "2024-01-15T10:00:00" 120 --use-opensearch\n' - ' collect_logs.py "2024-01-15T10:00:00" 60 --mode kubernetes\n' - ), - ) - parser.add_argument( - "start_time", - help=( - "Start of the collection window (UTC assumed if no timezone given). " - 'Formats: "2024-01-15T10:00:00" or "2024-01-15 10:00:00"' - ), - ) - parser.add_argument( - "duration_minutes", - type=int, - help="Duration in minutes.", - ) - parser.add_argument( - "--output-dir", - default=".", - metavar="DIR", - help="Directory to write the output tarball (default: current directory).", - ) - parser.add_argument( - "--mode", - choices=["docker", "kubernetes"], - default="docker", - help=( - "Deployment mode: 'docker' (default) uses Docker Swarm service names " - "for control-plane log collection; 'kubernetes' uses Kubernetes container " - "names and skips Graylog-based SPDK log collection (kubectl is used instead)." - ), - ) - parser.add_argument( - "--use-opensearch", - action="store_true", - help=( - "Query OpenSearch directly via scroll API instead of the Graylog " - "REST API. Useful for very large result sets or when Graylog is " - "unreachable." - ), - ) - parser.add_argument( - "--cluster-id", - metavar="UUID", - help="Target a specific cluster UUID (default: first cluster returned by sbctl).", - ) - parser.add_argument( - "--mgmt-ip", - metavar="IP", - help="Override the management-node IP used to reach Graylog / OpenSearch.", - ) - parser.add_argument( - "--monitoring-secret", - metavar="SECRET", - help=( - "Graylog / OpenSearch password to use instead of the cluster secret. " - "When provided this takes precedence over the cluster secret." - ), - ) - parser.add_argument( - "--namespace", - default="simplyblock", - metavar="NS", - help=( - "Kubernetes namespace to collect CSI / storage-node DS pod logs from " - "(default: simplyblock). Pass an empty string to skip kubectl collection." - ), - ) - parser.add_argument( - "--diagnose", - action="store_true", - help=( - "Print a diagnostic report from OpenSearch (indices, field names, " - "sample documents, container names present in the time window) and " - "exit without collecting logs. Use this when collections return 0 " - "to understand the actual data layout. Implies --use-opensearch." - ), - ) - args = parser.parse_args() - if args.diagnose: - args.use_opensearch = True - - # ── 1. Parse time range ────────────────────────────────────────────────── - - try: - start_dt = datetime.fromisoformat(args.start_time.replace(" ", "T")) - except ValueError as exc: - print(f"ERROR: invalid start_time – {exc}", file=sys.stderr) - sys.exit(1) - - if start_dt.tzinfo is None: - start_dt = start_dt.replace(tzinfo=timezone.utc) - - end_dt = start_dt + timedelta(minutes=args.duration_minutes) - from_iso = start_dt.strftime("%Y-%m-%dT%H:%M:%S.000Z") - to_iso = end_dt.strftime("%Y-%m-%dT%H:%M:%S.000Z") - - print("=" * 64) - print(" Simplyblock Log Collector") - print("=" * 64) - print(f" Window : {from_iso} → {to_iso} ({args.duration_minutes} min)") - print(f" Deploy : {args.mode}") - print(f" Mode : {'OpenSearch (direct)' if args.use_opensearch else 'Graylog REST API'}") - - # ── 2. Cluster UUID + secret ───────────────────────────────────────────── - - print("\n[1] Retrieving cluster info …") - cluster_uuid = args.cluster_id - if not cluster_uuid: - clusters = sbctl_json("cluster", "list") - if not clusters: - print("ERROR: 'sbctl cluster list' returned nothing.", file=sys.stderr) - sys.exit(1) - cluster_uuid = clusters[0]["UUID"] - - print(f" Cluster UUID : {cluster_uuid}") - - cluster_secret = sbctl_raw("cluster", "get-secret", cluster_uuid) - if not cluster_secret: - print("ERROR: could not retrieve cluster secret.", file=sys.stderr) - sys.exit(1) - print(f" Secret : {'*' * min(len(cluster_secret), 8)}… (len={len(cluster_secret)})") - - # ── 3. Management-node IP ──────────────────────────────────────────────── - - print("\n[2] Resolving management node …") - if args.mgmt_ip: - mgmt_ip = args.mgmt_ip - print(f" Using provided IP : {mgmt_ip}") - else: - cp_nodes = sbctl_json("control-plane", "list") - if not cp_nodes: - print("ERROR: 'sbctl control-plane list' returned nothing.", file=sys.stderr) - sys.exit(1) - mgmt_ip = cp_nodes[0]["IP"] - print(f" Management IP : {mgmt_ip} ({len(cp_nodes)} node(s) total)") - - if args.mode == "kubernetes": - graylog_base = f"http://{mgmt_ip}:9000/api" - opensearch_base = f"http://{mgmt_ip}:9200" - else: - graylog_base = f"http://{mgmt_ip}/graylog/api" - opensearch_base = f"http://{mgmt_ip}/opensearch" - - # ── 4. Storage nodes ───────────────────────────────────────────────────── - - print("\n[3] Retrieving storage nodes …") - sn_list = sbctl_json("storage-node", "list") or [] - if not sn_list: - print(" WARN: no storage nodes found (continuing without them).") - else: - print(f" Found {len(sn_list)} storage node(s).") - - # ── 5. HTTP sessions ───────────────────────────────────────────────────── - - graylog_password = args.monitoring_secret if args.monitoring_secret else cluster_secret - if args.monitoring_secret: - print(" Using provided --monitoring-secret for Graylog auth.") - - gl_session = requests.Session() - gl_session.auth = ("admin", graylog_password) - gl_session.headers.update({"X-Requested-By": "sb-log-collector"}) - - os_session = requests.Session() - - # Verify Graylog reachability (informational only) - if not args.use_opensearch: - print(f"\n[4] Checking Graylog at {graylog_base} …") - try: - r = gl_session.get(f"{graylog_base}/system", timeout=10) - if r.status_code == 200: - ver = r.json().get("version", "?") - print(f" OK (version {ver})") - else: - print(f" WARN: HTTP {r.status_code} – will still attempt collection.") - except requests.RequestException as exc: - print(f" WARN: {exc} – will still attempt collection.") - else: - print(f"\n[4] Checking OpenSearch at {opensearch_base} …") - try: - r = os_session.get(f"{opensearch_base}/_cluster/health", timeout=10) - if r.status_code == 200: - status = r.json().get("status", "?") - print(f" OK (cluster status: {status})") - else: - print(f" WARN: HTTP {r.status_code}.") - except requests.RequestException as exc: - print(f" WARN: {exc}.") - - # --diagnose: print full report and exit - if args.diagnose: - opensearch_diagnose(os_session, opensearch_base, from_iso, to_iso) - sys.exit(0) - - # ── 6. Prepare temp workspace ──────────────────────────────────────────── - - ts_str = start_dt.strftime("%Y%m%d_%H%M%S") - bundle_name = f"sb_logs_{ts_str}_{args.duration_minutes}m" - output_dir = Path(args.output_dir).resolve() - output_dir.mkdir(parents=True, exist_ok=True) - tarball_path = output_dir / f"{bundle_name}.tar.gz" - - probe_cache: dict = {} # shared across all OpenSearch calls in this run - - fetch_kw = dict( - gl_session=gl_session, - os_session=os_session, - graylog_base=graylog_base, - opensearch_base=opensearch_base, - use_opensearch=args.use_opensearch, - from_iso=from_iso, - to_iso=to_iso, - probe_cache=probe_cache, - ) - - with tempfile.TemporaryDirectory() as tmpdir: - log_root = Path(tmpdir) / bundle_name - log_root.mkdir() - - # ── 7. Control-plane logs ──────────────────────────────────────────── - - cp_services = ( - CONTROL_PLANE_SERVICES_KUBERNETES - if args.mode == "kubernetes" - else CONTROL_PLANE_SERVICES_DOCKER - ) - print(f"\n[5] Collecting control-plane logs ({len(cp_services)} services, mode={args.mode}) …") - cp_dir = log_root / "control_plane" - cp_dir.mkdir() - - gl_cname_field = "kubernetes_container_name" if args.mode == "kubernetes" else "container_name" - - total_cp_lines = 0 - for svc in cp_services: - out_f = cp_dir / f"{svc}.log" - gl_q = f'{gl_cname_field}:/.*{_gl_escape(svc)}.*/' - n = fetch( - gl_query=gl_q, - os_container=svc, - os_source=None, - out_path=out_f, - **fetch_kw, - ) - total_cp_lines += n - status = f"{n:>8,} lines" - print(f" {svc:<42} {status}") - - print(f" {'Control-plane total':<42} {total_cp_lines:>8,} lines") - - # Every control-plane source empty means the log pipeline is not - # delivering, not that nothing happened. Say so loudly and fall back to - # the Docker service logs, otherwise the run ends with "Done!" and a - # tarball full of 0-byte files (2026-08-10: an entire fail-over - # investigation had no service logs because of exactly this). - if total_cp_lines == 0: - print("\n !! WARNING: 0 control-plane log lines retrieved.") - print(" !! Graylog/OpenSearch is not returning data for this window.") - if args.mode == "docker": - _collect_docker_service_logs(cp_dir) - else: - print(" !! Check the log pipeline before trusting this bundle.") - - # ── 8. Storage-node logs ───────────────────────────────────────────── - # Docker mode: collect SPDK/SNodeAPI logs from Graylog/OpenSearch. - # Kubernetes mode: SPDK logs are captured via kubectl in step 9. - - if args.mode == "docker": - print("\n[6] Collecting storage-node logs (docker) …") - sn_root = log_root / "storage_nodes" - sn_root.mkdir() - - # SNodeAPI runs on every storage node under the same container name. - # Its GELF 'source' field is the Docker host hostname whose exact - # format varies by deployment and cannot be reliably derived from - # the management IP alone. Collect ALL SNodeAPI logs once (no - # source filter) into a shared file; each line contains src= - # so per-node filtering can be done with grep afterwards. - print("\n SNodeAPI (all nodes combined) …") - snode_api_log = sn_root / "SNodeAPI_all_nodes.log" - snode_api_count = fetch( - gl_query='container_name:"SNodeAPI"', - os_container="SNodeAPI", - os_source=None, - out_path=snode_api_log, - **fetch_kw, - ) - print(f" {'SNodeAPI (all nodes)':<42} {snode_api_count:>8,} lines") - print(" (filter by src= to isolate per-node logs)") - - for node in sn_list: - hostname = node.get("Hostname", "unknown") - node_ip = node.get("Management IP", "") - rpc_port = node.get("SPDK P", 8080) - - node_label = f"{hostname}_{node_ip}".strip("_") if node_ip else hostname - node_dir = sn_root / node_label - node_dir.mkdir() - - print(f"\n Node: {hostname} ip={node_ip} rpc_port={rpc_port}") - - # spdk_N and spdk_proxy_N are globally unique by RPC port number; - # no source filter needed. - spdk_containers = [ - (f"spdk_{rpc_port}", f"spdk_{rpc_port}.log"), - (f"spdk_proxy_{rpc_port}", f"spdk_proxy_{rpc_port}.log"), - ] - - for cname, fname in spdk_containers: - out_f = node_dir / fname - n = fetch( - gl_query=f'container_name:"{cname}"', - os_container=cname, - os_source=None, - out_path=out_f, - **fetch_kw, - ) - print(f" {cname:<42} {n:>8,} lines") - else: - print("\n[6] Collecting storage-node logs (kubernetes) …") - sn_root = log_root / "storage_nodes" - sn_root.mkdir() - - for node in sn_list: - hostname = node.get("Hostname", "unknown") - node_ip = node.get("Management IP", "") - rpc_port = node.get("SPDK P", 8080) - - node_label = f"{hostname}_{node_ip}".strip("_") if node_ip else hostname - node_dir = sn_root / node_label - node_dir.mkdir() - - print(f"\n Node: {hostname} ip={node_ip} rpc_port={rpc_port}") - - # Pod name pattern: snode-spdk-pod-- - # Container names inside that pod: spdk-container, spdk-proxy-container - pod_name = f"snode-spdk-pod-{rpc_port}-*" - spdk_containers = [ - ("spdk-container", f"spdk-container_{rpc_port}.log"), - ("spdk-proxy-container", f"spdk-proxy-container_{rpc_port}.log"), - ] - - for cname, fname in spdk_containers: - out_f = node_dir / fname - gl_q = ( - f'kubernetes_pod_name:{_gl_escape(pod_name)} ' - f'AND kubernetes_container_name:/.*{_gl_escape(cname)}.*/' - ) - n = fetch( - gl_query=gl_q, - os_container=cname, - os_source=None, - os_pod_name=pod_name, - out_path=out_f, - **fetch_kw, - ) - print(f" {cname:<42} {n:>8,} lines") - - # ── 9. Kubernetes pod logs (CSI node + storage-node DS) ────────────── - - k8s_ns = args.namespace - if k8s_ns: - print(f"\n[7] Collecting Kubernetes pod logs (namespace: {k8s_ns}) …") - k8s_dir = log_root / "k8s_pods" - k8s_dir.mkdir() - - # 9a. simplyblock-csi-node* pods — all containers + dmesg - csi_pods = _kubectl_list_pods(k8s_ns, "simplyblock-csi-node") - if csi_pods: - csi_dir = k8s_dir / "csi-node" - csi_dir.mkdir() - print(f" CSI node pods ({len(csi_pods)}) …") - for pod in csi_pods: - collect_k8s_pod_logs(k8s_ns, pod, csi_dir, from_iso, to_iso) - collect_k8s_csi_dmesg(k8s_ns, pod, csi_dir, from_iso, to_iso) - else: - print(f" No simplyblock-csi-node pods found in namespace {k8s_ns}.") - - # 9b. simplyblock-csi-controller* pods — all containers - csi_ctrl_pods = _kubectl_list_pods(k8s_ns, "simplyblock-csi-controller") - if csi_ctrl_pods: - csi_ctrl_dir = k8s_dir / "csi-controller" - csi_ctrl_dir.mkdir() - print(f" CSI controller pods ({len(csi_ctrl_pods)}) …") - for pod in csi_ctrl_pods: - collect_k8s_pod_logs(k8s_ns, pod, csi_ctrl_dir, from_iso, to_iso) - else: - print(f" No simplyblock-csi-controller pods found in namespace {k8s_ns}.") - - # 9c. simplyblock-manager* pods — all containers - mgr_pods = _kubectl_list_pods(k8s_ns, "simplyblock-manager") - if mgr_pods: - mgr_dir = k8s_dir / "simplyblock-manager" - mgr_dir.mkdir() - print(f" Simplyblock manager pods ({len(mgr_pods)}) …") - for pod in mgr_pods: - collect_k8s_pod_logs(k8s_ns, pod, mgr_dir, from_iso, to_iso) - else: - print(f" No simplyblock-manager pods found in namespace {k8s_ns}.") - - # 9d. simplyblock-storage-node-ds* pods — all containers - sn_ds_pods = _kubectl_list_pods(k8s_ns, "simplyblock-storage-node-ds") - if sn_ds_pods: - sn_ds_dir = k8s_dir / "storage-node-ds" - sn_ds_dir.mkdir() - print(f" Storage-node DS pods ({len(sn_ds_pods)}) …") - for pod in sn_ds_pods: - collect_k8s_pod_logs(k8s_ns, pod, sn_ds_dir, from_iso, to_iso) - else: - print(f" No simplyblock-storage-node-ds pods found in namespace {k8s_ns}.") - else: - print("\n[7] Skipping Kubernetes pod logs (--namespace not set).") - - # ── 10. sbctl cluster / node snapshots ─────────────────────────────── - - print("\n[8] Collecting sbctl cluster / node info …") - info_dir = log_root / "sbctl_info" - info_dir.mkdir() - - def save_sbctl(label, cmd_args, out_name, use_json=False): - """Run sbctl, save output to out_name, print status.""" - if use_json: - data = sbctl_json(*cmd_args) - if data is not None: - out_path = info_dir / out_name - with open(out_path, "w") as f: - json.dump(data, f, indent=2) - print(f" {label:<50} OK ({out_name})") - return True - else: - text = sbctl_raw(*cmd_args) - if text is not None: - out_path = info_dir / out_name - out_path.write_text(text) - print(f" {label:<50} OK ({out_name})") - return True - print(f" {label:<50} FAILED", file=sys.stderr) - return False - - # 1. cluster show - save_sbctl( - "sbctl cluster show", - ["cluster", "show", cluster_uuid], - "cluster_show.txt", - ) - - # 2. lvol list - save_sbctl( - "sbctl lvol list", - ["lvol", "list", "--cluster-id", cluster_uuid], - "lvol_list.json", - use_json=True, - ) - - # 3. sn list (already fetched; save the raw JSON for completeness) - save_sbctl( - "sbctl sn list", - ["sn", "list"], - "sn_list.json", - use_json=True, - ) - - # 4. sn check – one file per storage node - print(" sbctl sn check (per node) …") - sn_check_dir = info_dir / "sn_check" - sn_check_dir.mkdir() - for node in sn_list: - node_uuid = node.get("UUID", "") - node_hostname = node.get("Hostname", node_uuid) - node_ip = node.get("Management IP", "") - label = f"{node_hostname}_{node_ip}".strip("_") if node_ip else node_hostname - text = sbctl_raw("sn", "check", node_uuid) - if text is not None: - (sn_check_dir / f"{label}.txt").write_text(text) - print(f" {label}") - else: - print(f" {label} FAILED", file=sys.stderr) - - # 5. cluster get-logs --limit 0 (all cluster-level events) - save_sbctl( - "sbctl cluster get-logs --limit 0", - ["cluster", "get-logs", cluster_uuid, "--limit", "0"], - "cluster_get_logs.txt", - ) - - # ── 11. Write a collection manifest ────────────────────────────────── - - manifest = { - "collected_at": datetime.now(timezone.utc).isoformat(), - "window_from": from_iso, - "window_to": to_iso, - "duration_minutes": args.duration_minutes, - "cluster_uuid": cluster_uuid, - "mgmt_ip": mgmt_ip, - "deploy_mode": args.mode, - "log_source": "opensearch-direct" if args.use_opensearch else "graylog-api", - "storage_nodes": [ - { - "hostname": n.get("Hostname"), - "ip": n.get("Management IP"), - "rpc_port": n.get("SPDK P"), - "uuid": n.get("UUID"), - } - for n in sn_list - ], - } - with open(log_root / "manifest.json", "w") as mf: - json.dump(manifest, mf, indent=2) - - # ── 12. Pack into tarball ───────────────────────────────────────────── - - print("\n[9] Creating tarball …") - with tarfile.open(str(tarball_path), "w:gz") as tar: - tar.add(str(log_root), arcname=bundle_name) - - size_mb = tarball_path.stat().st_size / 1_048_576 - print(f"\n{'=' * 64}") - print(" Done!") - print(f" Tarball : {tarball_path}") - print(f" Size : {size_mb:.2f} MB") - print(f"{'=' * 64}\n") - - -if __name__ == "__main__": - main() +#!/usr/bin/env python3 +""" +Simplyblock Log Collector +========================= +Collects container logs from Graylog (or directly from OpenSearch) for a +specified time window, organises them by storage node and control-plane +service, and packages everything into a compressed tarball. + +The script must be run on a management node or inside an admin pod where +the `sbctl` CLI is available and has full admin access. + +Usage +----- + collect_logs.py [options] + + start_time ISO-8601 datetime, UTC assumed when no timezone given. + Accepted formats: "2024-01-15T10:00:00" + "2024-01-15 10:00:00" + "2024-01-15T10:00:00+00:00" + + duration_minutes Number of minutes to collect from start_time. + +Options +------- + --output-dir DIR Write the tarball here (default: current directory). + --mode MODE Deployment mode: "docker" (default) or "kubernetes". + Selects the set of control-plane service names and + adjusts which log sources are queried. + --use-opensearch Query OpenSearch scroll API directly instead of the + Graylog search REST API. Useful when Graylog is + unavailable or when the result set is very large. + --cluster-id UUID Force a specific cluster UUID (default: first cluster). + --mgmt-ip IP Override management-node IP for Graylog / OpenSearch. + +Examples +-------- + collect_logs.py "2024-01-15T10:00:00" 60 + collect_logs.py "2024-01-15 10:00:00" 30 --output-dir /tmp/logs + collect_logs.py "2024-01-15T10:00:00" 120 --use-opensearch + collect_logs.py "2024-01-15T10:00:00" 60 --mode kubernetes +""" + +import argparse +import json +import os +import shutil +import subprocess +import sys +import tarfile +import tempfile +from datetime import datetime, timezone, timedelta +from pathlib import Path +from typing import Any + +try: + import requests +except ImportError: + print( + "ERROR: the 'requests' library is required.\n" + " Install it with: pip3 install requests", + file=sys.stderr, + ) + sys.exit(1) + +# --------------------------------------------------------------------------- +# Constants +# --------------------------------------------------------------------------- + +# Maximum records per single Graylog search page. +PAGE_SIZE = 1000 + +# OpenSearch max_result_window is set to 100 000 during cluster initialisation +# (see simplyblock_core/cluster_ops.py :: _set_max_result_window). +# Requests that would exceed this threshold are split into time-based chunks. +MAX_RESULT_WINDOW = 100_000 + +# Docker Swarm service names that run on the management / control-plane node. +CONTROL_PLANE_SERVICES_DOCKER = [ + "WebAppAPI", + "WebAppAPI2", + "WebAppAPI3", + "WebAppAPI4", + "WebAppAPI5", + "fdb-server", + "fdb-backup-agent", + "StorageNodeMonitor", + "MgmtNodeMonitor", + "LVolStatsCollector", + "MainDistrEventCollector", + "CapacityAndStatsCollector", + "CapacityMonitor", + "HealthCheck", + "DeviceMonitor", + "LVolMonitor", + "SnapshotMonitor", + "TasksRunnerRestart", + "TasksRunnerMigration", + "TasksRunnerLVolMigration", + "TasksRunnerFailedMigration", + "TasksRunnerClusterStatus", + "TasksRunnerNewDeviceMigration", + "TasksNodeAddRunner", + "TasksRunnerClusterExpand", + "TasksRunnerPortAllow", + "TasksRunnerJCCompResume", + "TasksRunnerLVolSyncDelete", + "TasksRunnerBackup", + "BackupMergeService", + # Async cross-cluster replication: these run on the CP but were missing from + # this list, so a replication incident collected no replication logs at all. + "SnapshotReplication", + "TasksRunnerReplicationFinal", + "TasksRunnerBatchMigration", + "TasksNodeRemovalRunner", + "HAProxy", +] + +CONTROL_PLANE_SERVICES_KUBERNETES = [ + "simplyblock-control", + "webappapi", + "storage-node-monitor", + "mgmt-node-monitor", + "lvol-stats-collector", + "main-distr-event-collector", + "capacity-and-stats-collector", + "capacity-monitor", + "health-check", + "device-monitor", + "lvol-monitor", + "snapshot-monitor", + "tasks-node-add-runner", + "tasks-runner-restart", + "tasks-runner-migration", + "tasks-runner-failed-migration", + "tasks-runner-cluster-status", + "tasks-runner-new-device-migration", + "tasks-runner-port-allow", + "tasks-runner-jc-comp-resume", + "tasks-runner-sync-lvol-del", + "tasks-runner-backup", + "tasks-runner-backup-merge", + "tasks-runner-snapshot-replication", + "tasks-runner-replication-final", + "tasks-runner-batch-migration", +] + +# --------------------------------------------------------------------------- +# sbctl helpers +# --------------------------------------------------------------------------- + + +def _sbctl_bin(): + """Absolute path to the sbctl binary. + + The script is normally run under sudo, whose secure_path does not include + /usr/local/bin, so a bare "sbctl" raises FileNotFoundError and the collector + exits before gathering anything. + """ + found = shutil.which("sbctl") + if found: + return found + for candidate in ("/usr/local/bin/sbctl", "/usr/bin/sbctl"): + if os.path.isfile(candidate) and os.access(candidate, os.X_OK): + return candidate + return "sbctl" + + +SBCTL_BIN = _sbctl_bin() + + +def _collect_docker_service_logs(cp_dir, tail=40000): + """Fallback: pull control-plane logs straight from Docker Swarm. + + Used when Graylog/OpenSearch returns nothing. `docker service logs` + aggregates ALL tasks of a service, including ones that were replaced during + the window — `docker logs ` only has the currently running task, + so a service recreated mid-incident loses its earlier output entirely. + + Best-effort: this only works on a docker-mode management node, and any + failure is reported rather than raised so the rest of the bundle survives. + """ + print(" -> falling back to `docker service logs` on this node") + r = _run(["docker", "service", "ls", "--format", "{{.Name}}"], timeout=60) + if r is None or r.returncode != 0: + print(" !! cannot list docker services; no control-plane logs in this bundle") + return + services = [s.strip() for s in r.stdout.splitlines() if s.strip()] + if not services: + print(" !! no docker services found") + return + + out_dir = cp_dir / "docker_service_logs" + out_dir.mkdir(exist_ok=True) + total = 0 + for svc in services: + r = _run(["docker", "service", "logs", "--timestamps", "--no-task-ids", + "--tail", str(tail), svc], timeout=600) + if r is None: + print(f" {svc:<42} {'timed out':>8}") + continue + body = (r.stdout or "") + (r.stderr or "") + (out_dir / f"{svc}.log").write_text(body, encoding="utf-8", errors="replace") + lines = body.count("\n") + total += lines + print(f" {svc:<42} {lines:>8,} lines") + print(f" {'Docker service logs total':<42} {total:>8,} lines") + + +def _run(cmd, timeout=30): + """Run *cmd* list; return CompletedProcess or None on failure.""" + try: + return subprocess.run(cmd, capture_output=True, text=True, timeout=timeout) + except FileNotFoundError: + print(f"ERROR: command not found: {cmd[0]}", file=sys.stderr) + sys.exit(1) + except subprocess.TimeoutExpired: + print(f"ERROR: command timed out: {' '.join(cmd)}", file=sys.stderr) + return None + + +def sbctl_json(*args): + """ + Run ``sbctl --json`` and return the parsed JSON (list or dict). + Returns None and prints an error on failure. + """ + cmd = [SBCTL_BIN] + list(args) + ["--json"] + r = _run(cmd) + if r is None or r.returncode != 0: + if r: + print(f"ERROR: {' '.join(cmd)}\n stderr: {r.stderr.strip()}", file=sys.stderr) + return None + try: + return json.loads(r.stdout) + except json.JSONDecodeError: + print( + f"ERROR: could not parse JSON from: {' '.join(cmd)}\n" + f" output: {r.stdout[:400]}", + file=sys.stderr, + ) + return None + + +def sbctl_raw(*args): + """ + Run ``sbctl `` (no --json) and return stripped stdout text. + Returns None on failure. + """ + r = _run([SBCTL_BIN] + list(args)) + if r is None or r.returncode != 0: + if r: + print( + f"ERROR: sbctl {' '.join(args)}\n stderr: {r.stderr.strip()}", + file=sys.stderr, + ) + return None + return r.stdout.strip() + + +# --------------------------------------------------------------------------- +# Log-line formatter +# --------------------------------------------------------------------------- + + +def _fmt(msg: dict) -> str: + """Render a Graylog / OpenSearch message dict as a single log line.""" + ts = msg.get("timestamp", "") + src = msg.get("source", "") + cname = msg.get("container_name", "") + lvl = msg.get("level", "") + text = str(msg.get("message", "")).replace("\n", "\\n") + return f"{ts} src={src} ctr={cname} lvl={lvl} {text}" + + +# --------------------------------------------------------------------------- +# Graylog REST API helpers +# --------------------------------------------------------------------------- + +def _gl_escape(value: str) -> str: + """ + Escape Lucene special characters in a Graylog field query term. + Hyphens are NOT escaped — they are only special in range expressions + and cause HTTP 400 when escaped in the Graylog REST API. + """ + return value.replace(".", "\\.") + + +def _gl_search_page(session, search_url, query, from_iso, to_iso, limit, offset): + """ + Fetch one page of results from the Graylog absolute-search endpoint. + Returns (messages_list, total_results) or (None, 0) on error. + """ + params = { + "query": query, + "from": from_iso, + "to": to_iso, + "limit": limit, + "offset": offset, + "sort": "timestamp:asc", + "fields": "timestamp,source,container_name,level,message", + } + try: + resp = session.get(search_url, params=params, timeout=90, + headers={"Accept": "application/json"}) + resp.raise_for_status() + except requests.RequestException as exc: + print(f" WARN: Graylog page request failed (offset={offset}): {exc}", file=sys.stderr) + return None, 0 + + if not resp.text.strip(): + print(f" WARN: Graylog returned empty response (offset={offset}, status={resp.status_code})", file=sys.stderr) + return None, 0 + try: + data = resp.json() + except requests.exceptions.JSONDecodeError as exc: + print(f" WARN: Graylog response is not valid JSON (offset={offset}): {exc}", file=sys.stderr) + return None, 0 + return data.get("messages", []), data.get("total_results", 0) + + +def _gl_write_window(session, search_url, query, from_iso, to_iso, fh): + """ + Paginate through a single time window and write ALL lines to *fh*. + + The window MUST have total <= MAX_RESULT_WINDOW. Caller is responsible + for checking the count first (via ``_gl_probe_total``) and bisecting + the window when it exceeds the limit. + + Returns number of lines written. + """ + written = 0 + offset = 0 + + while True: + msgs, _ = _gl_search_page( + session, search_url, query, from_iso, to_iso, PAGE_SIZE, offset + ) + if msgs is None: + break + if not msgs: + break + for m in msgs: + fh.write(_fmt(m.get("message", {})) + "\n") + written += 1 + offset += len(msgs) + if len(msgs) < PAGE_SIZE: + break + if offset >= MAX_RESULT_WINDOW: + break + + return written + + +def _gl_probe_total(session, search_url, query, from_iso, to_iso): + """Return total number of matching entries for a window, or -1 on error.""" + msgs, total = _gl_search_page(session, search_url, query, from_iso, to_iso, 1, 0) + if msgs is None: + return -1 + return total + + +# Minimum bisection window size in seconds. Windows smaller than this are +# fetched best-effort (capped at MAX_RESULT_WINDOW). +MIN_BISECT_SEC = 1 + + +def _gl_fetch_recursive(session, search_url, query, from_dt, to_dt, fh, + depth=0): + """ + Recursively bisect the time window until entries fit within + MAX_RESULT_WINDOW, then paginate and write. + + Returns ``(lines_written, truncated_count)`` where *truncated_count* + is the number of leaf windows that exceeded the limit even at the + minimum window size (data loss). + """ + from_iso = from_dt.strftime("%Y-%m-%dT%H:%M:%S.000Z") + to_iso = to_dt.strftime("%Y-%m-%dT%H:%M:%S.000Z") + window_sec = (to_dt - from_dt).total_seconds() + indent = " " + " " * min(depth, 6) + + total = _gl_probe_total(session, search_url, query, from_iso, to_iso) + if total <= 0: + return 0, 0 + + # Window fits — fetch everything + if total <= MAX_RESULT_WINDOW: + written = _gl_write_window(session, search_url, query, + from_iso, to_iso, fh) + return written, 0 + + # Window too big — can we bisect further? + if window_sec <= MIN_BISECT_SEC: + # Smallest window reached — best-effort fetch (capped at 100k) + print(f"{indent}WARN: {window_sec:.0f}s window at {from_iso} has " + f"{total} entries (>{MAX_RESULT_WINDOW}), capturing first " + f"{MAX_RESULT_WINDOW} (best effort)", file=sys.stderr) + written = _gl_write_window(session, search_url, query, + from_iso, to_iso, fh) + return written, 1 + + # Bisect + mid_dt = from_dt + (to_dt - from_dt) / 2 + print(f"{indent}NOTE: {from_iso}..{to_iso} ({window_sec:.0f}s) has " + f"{total} entries, bisecting") + + w1, t1 = _gl_fetch_recursive(session, search_url, query, + from_dt, mid_dt, fh, depth + 1) + w2, t2 = _gl_fetch_recursive(session, search_url, query, + mid_dt, to_dt, fh, depth + 1) + return w1 + w2, t1 + t2 + + +def graylog_fetch_all(session, base_url, query, from_iso, to_iso, out_path): + """ + Download all log messages matching *query* within [from_iso, to_iso]. + + Uses recursive binary bisection: probe the window count first, and if + it exceeds MAX_RESULT_WINDOW, split the window in half and recurse. + Keeps halving down to MIN_BISECT_SEC (1 second). Only writes data at + leaf windows that fit within the limit (no duplicate writes). + + Writes one text line per message to *out_path*. + Returns ``(lines_written, truncated_count)`` where *truncated_count* + is the number of leaf windows that still exceeded the limit at the + minimum window size. + """ + search_url = f"{base_url}/search/universal/absolute" + + # Quick probe + total = _gl_probe_total(session, search_url, query, from_iso, to_iso) + if total < 0: + Path(out_path).touch() + return 0, 0 + if total == 0: + Path(out_path).touch() + print(" total entries: 0") + return 0, 0 + + print(f" total entries: {total}") + + from_dt = datetime.fromisoformat(from_iso.replace("Z", "+00:00")) + to_dt = datetime.fromisoformat(to_iso.replace("Z", "+00:00")) + + with open(out_path, "w") as fh: + written, truncated = _gl_fetch_recursive( + session, search_url, query, from_dt, to_dt, fh, + ) + + return written, truncated + + +# --------------------------------------------------------------------------- +# OpenSearch scroll API helpers (--use-opensearch) +# --------------------------------------------------------------------------- + + +def _os_get_index(session, os_url): + """ + Discover the graylog indices present in OpenSearch and return them as a + comma-separated string suitable for use in a URL path segment. + + Using _cat/indices avoids embedding a '*' wildcard in the URL, which + HAProxy may reject (400). Falls back to '_all' if discovery fails. + """ + try: + r = session.get(f"{os_url}/_cat/indices?h=index&format=json", timeout=10) + r.raise_for_status() + indices = sorted( + i["index"] + for i in r.json() + if i["index"].startswith("graylog") and not i["index"].startswith(".") + ) + if indices: + return ",".join(indices) + except Exception as exc: + print(f" WARN: could not discover OpenSearch indices ({exc}); using _all", file=sys.stderr) + return "_all" + + +def _os_probe(session, os_url, index, from_ms, to_ms): + """ + Probe the index to discover: + - The actual timestamp field name (e.g. 'timestamp' vs '@timestamp') + - The actual container-name field name + - How many documents exist in the requested time window (any container) + - A sample document so we can see real field values + + Returns a dict with keys: ts_field, cname_field, window_count, sample_doc + """ + result = {"ts_field": "timestamp", "cname_field": "container_name", + "window_count": 0, "sample_doc": None} + + # --- sample document (no time filter) --- + try: + r = session.post( + f"{os_url}/{index}/_search", + json={"size": 1, "query": {"match_all": {}}}, + timeout=10, + ) + if r.ok: + hits = r.json().get("hits", {}).get("hits", []) + if hits: + src = hits[0].get("_source", {}) + result["sample_doc"] = src + # Detect timestamp field + if "@timestamp" in src: + result["ts_field"] = "@timestamp" + # Detect container-name field (various naming conventions) + for candidate in ("kubernetes_container_name", "container_name", + "container_id", "containerName", + "_container_name", "docker_container_name"): + if candidate in src: + result["cname_field"] = candidate + break + except Exception as exc: + print(f" WARN: probe (sample doc) failed: {exc}", file=sys.stderr) + + # --- count within the requested time window --- + ts = result["ts_field"] + try: + r = session.post( + f"{os_url}/{index}/_count", + json={"query": {"range": {ts: {"gte": from_ms, "lte": to_ms, + "format": "epoch_millis"}}}}, + timeout=10, + ) + if r.ok: + result["window_count"] = r.json().get("count", 0) + except Exception as exc: + print(f" WARN: probe (window count) failed: {exc}", file=sys.stderr) + + return result + + +def _os_sample_container_names(session, os_url, index, from_ms, to_ms, ts_field, cname_field, n=30): + """ + Return up to *n* distinct container_name values within the time window + using a terms aggregation. Used by --diagnose. + """ + body = { + "size": 0, + "query": {"range": {ts_field: {"gte": from_ms, "lte": to_ms, + "format": "epoch_millis"}}}, + "aggs": { + "names": { + "terms": { + "field": f"{cname_field}.keyword", + "size": n, + } + } + }, + } + try: + r = session.post(f"{os_url}/{index}/_search", json=body, timeout=15) + if r.ok: + buckets = r.json().get("aggregations", {}).get("names", {}).get("buckets", []) + return [(b["key"], b["doc_count"]) for b in buckets] + except Exception: + pass + return [] + + +def opensearch_diagnose(session, os_url, from_iso, to_iso): + """ + Print a detailed diagnostic report about what is in OpenSearch. + Called when --diagnose is passed. + """ + print("\n" + "=" * 64) + print(" OpenSearch Diagnostic Report") + print("=" * 64) + + from_ms = int(datetime.fromisoformat(from_iso.replace("Z", "+00:00")).timestamp() * 1000) + to_ms = int(datetime.fromisoformat(to_iso.replace("Z", "+00:00")).timestamp() * 1000) + + # 1. List all indices + print("\n[D1] All indices:") + try: + r = session.get(f"{os_url}/_cat/indices?h=index,docs.count,store.size&format=json", + timeout=10) + r.raise_for_status() + for idx in sorted(r.json(), key=lambda x: x["index"]): + print(f" {idx['index']:<45} docs={idx.get('docs.count','?'):>10} " + f"size={idx.get('store.size','?')}") + except Exception as exc: + print(f" ERROR: {exc}") + + index = _os_get_index(session, os_url) + print(f"\n → Using index(es): {index}") + + # 2. Probe + probe = _os_probe(session, os_url, index, from_ms, to_ms) + print("\n[D2] Detected field names:") + print(f" timestamp field : {probe['ts_field']}") + print(f" container_name field: {probe['cname_field']}") + print(f"\n[D3] Documents in requested time window: {probe['window_count']}") + + # 3. Sample document + if probe["sample_doc"]: + print("\n[D4] Sample document fields and values:") + for k, v in sorted(probe["sample_doc"].items()): + v_str = str(v)[:120] + print(f" {k:<35} = {v_str}") + else: + print("\n[D4] No sample document found (index may be empty).") + + # 4. Container names in window + print("\n[D5] Distinct container_name values in time window (up to 30):") + names = _os_sample_container_names(session, os_url, index, + from_ms, to_ms, + probe["ts_field"], probe["cname_field"]) + if names: + for name, count in names: + print(f" {name:<60} {count:>8} docs") + else: + print(" (none found – aggregation on .keyword sub-field may have failed)") + print(" Trying match_all sample …") + try: + r = session.post( + f"{os_url}/{index}/_search", + json={"size": 5, "query": {"match_all": {}}, + "_source": [probe["cname_field"]]}, + timeout=10, + ) + if r.ok: + for h in r.json().get("hits", {}).get("hits", []): + print(f" {h.get('_source', {}).get(probe['cname_field'], '???')}") + except Exception: + pass + + print("\n" + "=" * 64) + + +def opensearch_fetch_all(session, os_url, container_name, source, from_iso, to_iso, out_path, + probe_cache=None, pod_name=None): + """ + Fetch logs directly from OpenSearch using the scroll API. + + Discovers the actual timestamp and container-name field names via a + one-time probe (cached in *probe_cache* dict across calls). + Uses query_string wildcards for container matching so Docker Swarm + names like 'simplyblock_WebAppAPI.1.' are matched by just + passing 'WebAppAPI'. + Returns number of lines written. + """ + # Graylog's OpenSearch index maps the timestamp field with format + # "uuuu-MM-dd HH:mm:ss.SSS" (space separator, no timezone suffix). + # epoch_millis is accepted regardless of the field's stored date format. + from_ms = int(datetime.fromisoformat(from_iso.replace("Z", "+00:00")).timestamp() * 1000) + to_ms = int(datetime.fromisoformat(to_iso.replace("Z", "+00:00")).timestamp() * 1000) + + # One-time index discovery + probe (cached) + if probe_cache is None: + probe_cache = {} + if "index" not in probe_cache: + probe_cache["index"] = _os_get_index(session, os_url) + probe_cache["probe"] = _os_probe(session, os_url, probe_cache["index"], from_ms, to_ms) + p = probe_cache["probe"] + print(f" [OpenSearch] index={probe_cache['index']} " + f"ts_field={p['ts_field']} cname_field={p['cname_field']} " + f"docs_in_window={p['window_count']}") + if p["window_count"] == 0: + print(" WARN: no documents in the requested time window – " + "check the start_time / duration, or run with --diagnose", + file=sys.stderr) + + index = probe_cache["index"] + probe = probe_cache["probe"] + ts_f = probe["ts_field"] + cname_f = probe["cname_field"] + + # Build query + # Use query_string wildcards so partial names work: + # "WebAppAPI" matches "simplyblock_WebAppAPI.1.abc123" + # "spdk_8080" matches "/spdk_8080" + must_clauses: list[Any] = [ + {"range": {ts_f: {"gte": from_ms, "lte": to_ms, "format": "epoch_millis"}}}, + ] + if container_name: + esc = container_name.replace("/", "\\/").replace(":", "\\:") + must_clauses.append({ + "query_string": { + "default_field": cname_f, + "query": f"*{esc}*", + "analyze_wildcard": True, + } + }) + if pod_name: + esc_pod = pod_name.replace("/", "\\/").replace(":", "\\:") + must_clauses.append({ + "query_string": { + "default_field": "kubernetes_pod_name", + "query": f"*{esc_pod}*", + "analyze_wildcard": True, + } + }) + if source: + # source may be a single string or a list of candidate values + # (e.g. multiple hostname formats for the same node). + # When it is a list we OR them so any matching format succeeds. + candidates = source if isinstance(source, (list, tuple)) else [source] + if len(candidates) == 1: + must_clauses.append({ + "query_string": { + "default_field": "source", + "query": f'"{candidates[0]}"', + } + }) + else: + must_clauses.append({ + "bool": { + "should": [ + {"query_string": {"default_field": "source", + "query": f'"{c}"'}} + for c in candidates + ], + "minimum_should_match": 1, + } + }) + + body = { + "query": {"bool": {"must": must_clauses}}, + "sort": [{ts_f: {"order": "asc"}}], + "size": PAGE_SIZE, + "_source": [ts_f, "source", cname_f, "level", "message"], + } + + init_url = f"{os_url}/{index}/_search?scroll=2m" + written = 0 + + try: + r = session.post(init_url, json=body, timeout=60) + if not r.ok: + print( + f" WARN: OpenSearch initial scroll failed: {r.status_code} {r.reason}" + f"\n body: {r.text[:400]}", + file=sys.stderr, + ) + Path(out_path).touch() + return 0 + except requests.RequestException as exc: + print(f" WARN: OpenSearch initial scroll failed: {exc}", file=sys.stderr) + Path(out_path).touch() + return 0 + + data = r.json() + scroll_id = data.get("_scroll_id") + hits = data.get("hits", {}).get("hits", []) + total = data.get("hits", {}).get("total", {}) + total = total.get("value", total) if isinstance(total, dict) else int(total or 0) + print(f" total entries: {total}") + + with open(out_path, "w") as fh: + while hits: + for h in hits: + src = h.get("_source", {}) + # normalise field names to what _fmt expects + if ts_f != "timestamp": + src["timestamp"] = src.get(ts_f, "") + if cname_f != "container_name": + src["container_name"] = src.get(cname_f, "") + fh.write(_fmt(src) + "\n") + written += 1 + if len(hits) < PAGE_SIZE or not scroll_id: + break + try: + sc_r = session.post( + f"{os_url}/_search/scroll", + json={"scroll": "2m", "scroll_id": scroll_id}, + timeout=60, + ) + sc_r.raise_for_status() + sc_data = sc_r.json() + scroll_id = sc_data.get("_scroll_id", scroll_id) + hits = sc_data.get("hits", {}).get("hits", []) + except requests.RequestException as exc: + print(f" WARN: scroll continuation failed: {exc}", file=sys.stderr) + break + + # Release scroll context + if scroll_id: + try: + session.delete( + f"{os_url}/_search/scroll", + json={"scroll_id": scroll_id}, + timeout=10, + ) + except Exception: + pass + + return written + + +# --------------------------------------------------------------------------- +# Dispatch helper +# --------------------------------------------------------------------------- + + +def fetch( + *, + gl_session, + os_session, + graylog_base, + opensearch_base, + use_opensearch, + gl_query, + os_container, + os_source, + from_iso, + to_iso, + out_path, + probe_cache, + os_pod_name=None, +): + """Route to Graylog or OpenSearch depending on *use_opensearch*. + + When using Graylog and the recursive bisection still has truncated + leaf windows, automatically falls back to OpenSearch scroll API + (which has no offset limit) for the entire service. + """ + if use_opensearch: + return opensearch_fetch_all( + os_session, opensearch_base, + os_container, os_source, + from_iso, to_iso, str(out_path), + probe_cache=probe_cache, + pod_name=os_pod_name, + ) + + written, truncated = graylog_fetch_all( + gl_session, graylog_base, + gl_query, from_iso, to_iso, str(out_path), + ) + + if truncated and opensearch_base: + print(f" NOTE: Graylog had {truncated} truncated window(s), " + f"retrying via OpenSearch scroll API for complete data") + try: + os_written = opensearch_fetch_all( + os_session, opensearch_base, + os_container, os_source, + from_iso, to_iso, str(out_path), + probe_cache=probe_cache, + pod_name=os_pod_name, + ) + if os_written > 0: + return os_written + print(f" WARN: OpenSearch fallback returned 0 lines, " + f"keeping Graylog result ({written} lines)") + except Exception as exc: + print(f" WARN: OpenSearch fallback failed ({exc}), " + f"keeping Graylog result ({written} lines)", + file=sys.stderr) + + return written + + +# --------------------------------------------------------------------------- +# kubectl pod-log helpers +# --------------------------------------------------------------------------- + + +def _kubectl(*args, timeout=60) -> str: + """Run kubectl with the given args and return stdout. Returns '' on failure.""" + try: + r = subprocess.run( + ["kubectl"] + list(args), + capture_output=True, text=True, timeout=timeout, + ) + return r.stdout + except Exception as exc: + print(f" WARN: kubectl {' '.join(args[:4])} … failed: {exc}", file=sys.stderr) + return "" + + +def _kubectl_list_pods(namespace: str, prefix: str) -> list[str]: + """Return pod names in *namespace* whose name starts with *prefix*.""" + out = _kubectl("get", "pods", "-n", namespace, + "--no-headers", "-o", "custom-columns=:metadata.name") + return [p for p in out.splitlines() if p.startswith(prefix)] + + +def _kubectl_containers(namespace: str, pod: str) -> list[str]: + """Return init + regular container names for *pod*.""" + out = _kubectl( + "get", "pod", pod, "-n", namespace, + "-o", + "jsonpath={range .spec.initContainers[*]}{.name}{'\\n'}{end}" + "{range .spec.containers[*]}{.name}{'\\n'}{end}", + ) + return [c for c in out.splitlines() if c] + + +def collect_k8s_pod_logs(namespace: str, pod: str, out_dir: Path, + from_iso: str, to_iso: str) -> None: + """ + Write current + previous logs for every container in *pod* to *out_dir*. + Files are named _.log + """ + containers = _kubectl_containers(namespace, pod) + for container in containers: + log_file = out_dir / f"{pod}_{container}.log" + print(f" {pod} / {container}") + with open(log_file, "w") as fh: + fh.write(f"=== Pod: {pod} | Container: {container} | Namespace: {namespace} ===\n") + fh.write(f"=== From: {from_iso} | Until: {to_iso} ===\n\n") + + fh.write("--- current logs ---\n") + out = _kubectl("logs", pod, "-c", container, "-n", namespace, + "--timestamps", f"--since-time={from_iso}", timeout=120) + # Trim lines beyond to_iso + for line in out.splitlines(): + if line[:26] > to_iso[:26]: + break + fh.write(line + "\n") + + fh.write("\n--- previous (last crash) logs ---\n") + prev = _kubectl("logs", pod, "-c", container, "-n", namespace, + "--timestamps", "--previous", timeout=60) + fh.write(prev if prev.strip() else "(no previous logs)\n") + + +def collect_k8s_csi_dmesg(namespace: str, pod: str, out_dir: Path, + from_iso: str, to_iso: str) -> None: + """ + Collect dmesg from the csi-node container of a CSI pod, + filtered to the requested time window using the kernel boot epoch. + """ + from_epoch = int(datetime.fromisoformat(from_iso.replace("Z", "+00:00")).timestamp()) + to_epoch = int(datetime.fromisoformat(to_iso.replace("Z", "+00:00")).timestamp()) + + log_file = out_dir / f"{pod}_csi-node_dmesg.log" + print(f" {pod} / csi-node (dmesg)") + + # Derive boot epoch from /proc/uptime inside the container + uptime_out = _kubectl("exec", pod, "-c", "csi-node", "-n", namespace, + "--", "cat", "/proc/uptime", timeout=10) + try: + boot_epoch = int(datetime.now(timezone.utc).timestamp()) - int(float(uptime_out.split()[0])) + except Exception: + boot_epoch = 0 + + # Prefer human-readable reltime; fall back to monotonic seconds + dmesg_out = _kubectl("exec", pod, "-c", "csi-node", "-n", namespace, + "--", "dmesg", "--kernel", "--time-format=reltime", + "--nopager", timeout=30) + if not dmesg_out.strip(): + dmesg_out = _kubectl("exec", pod, "-c", "csi-node", "-n", namespace, + "--", "dmesg", "--kernel", "--nopager", timeout=30) + # Filter by monotonic timestamp + filtered = [] + import re + for line in dmesg_out.splitlines(): + m = re.match(r'^\[\s*([0-9]+\.[0-9]+)\]', line) + if m: + wall = boot_epoch + int(float(m.group(1))) + if wall < from_epoch: + continue + if wall > to_epoch: + break + filtered.append(line) + dmesg_out = "\n".join(filtered) + + with open(log_file, "w") as fh: + fh.write(f"=== Pod: {pod} | Container: csi-node | dmesg ===\n") + fh.write(f"=== From: {from_iso} | Until: {to_iso} ===\n\n") + fh.write(dmesg_out or "(no dmesg output)\n") + + +# --------------------------------------------------------------------------- +# Main +# --------------------------------------------------------------------------- + + +def main(): + parser = argparse.ArgumentParser( + prog="collect_logs.py", + description="Collect simplyblock container logs for a given time window.", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=( + "Examples:\n" + ' collect_logs.py "2024-01-15T10:00:00" 60\n' + ' collect_logs.py "2024-01-15 10:00:00" 30 --output-dir /tmp/logs\n' + ' collect_logs.py "2024-01-15T10:00:00" 120 --use-opensearch\n' + ' collect_logs.py "2024-01-15T10:00:00" 60 --mode kubernetes\n' + ), + ) + parser.add_argument( + "start_time", + help=( + "Start of the collection window (UTC assumed if no timezone given). " + 'Formats: "2024-01-15T10:00:00" or "2024-01-15 10:00:00"' + ), + ) + parser.add_argument( + "duration_minutes", + type=int, + help="Duration in minutes.", + ) + parser.add_argument( + "--output-dir", + default=".", + metavar="DIR", + help="Directory to write the output tarball (default: current directory).", + ) + parser.add_argument( + "--mode", + choices=["docker", "kubernetes"], + default="docker", + help=( + "Deployment mode: 'docker' (default) uses Docker Swarm service names " + "for control-plane log collection; 'kubernetes' uses Kubernetes container " + "names and skips Graylog-based SPDK log collection (kubectl is used instead)." + ), + ) + parser.add_argument( + "--use-opensearch", + action="store_true", + help=( + "Query OpenSearch directly via scroll API instead of the Graylog " + "REST API. Useful for very large result sets or when Graylog is " + "unreachable." + ), + ) + parser.add_argument( + "--cluster-id", + metavar="UUID", + help="Target a specific cluster UUID (default: first cluster returned by sbctl).", + ) + parser.add_argument( + "--mgmt-ip", + metavar="IP", + help="Override the management-node IP used to reach Graylog / OpenSearch.", + ) + parser.add_argument( + "--monitoring-secret", + metavar="SECRET", + help=( + "Graylog / OpenSearch password to use instead of the cluster secret. " + "When provided this takes precedence over the cluster secret." + ), + ) + parser.add_argument( + "--namespace", + default="simplyblock", + metavar="NS", + help=( + "Kubernetes namespace to collect CSI / storage-node DS pod logs from " + "(default: simplyblock). Pass an empty string to skip kubectl collection." + ), + ) + parser.add_argument( + "--diagnose", + action="store_true", + help=( + "Print a diagnostic report from OpenSearch (indices, field names, " + "sample documents, container names present in the time window) and " + "exit without collecting logs. Use this when collections return 0 " + "to understand the actual data layout. Implies --use-opensearch." + ), + ) + args = parser.parse_args() + if args.diagnose: + args.use_opensearch = True + + # ── 1. Parse time range ────────────────────────────────────────────────── + + try: + start_dt = datetime.fromisoformat(args.start_time.replace(" ", "T")) + except ValueError as exc: + print(f"ERROR: invalid start_time – {exc}", file=sys.stderr) + sys.exit(1) + + if start_dt.tzinfo is None: + start_dt = start_dt.replace(tzinfo=timezone.utc) + + end_dt = start_dt + timedelta(minutes=args.duration_minutes) + from_iso = start_dt.strftime("%Y-%m-%dT%H:%M:%S.000Z") + to_iso = end_dt.strftime("%Y-%m-%dT%H:%M:%S.000Z") + + print("=" * 64) + print(" Simplyblock Log Collector") + print("=" * 64) + print(f" Window : {from_iso} → {to_iso} ({args.duration_minutes} min)") + print(f" Deploy : {args.mode}") + print(f" Mode : {'OpenSearch (direct)' if args.use_opensearch else 'Graylog REST API'}") + + # ── 2. Cluster UUID + secret ───────────────────────────────────────────── + + print("\n[1] Retrieving cluster info …") + cluster_uuid = args.cluster_id + if not cluster_uuid: + clusters = sbctl_json("cluster", "list") + if not clusters: + print("ERROR: 'sbctl cluster list' returned nothing.", file=sys.stderr) + sys.exit(1) + cluster_uuid = clusters[0]["UUID"] + + print(f" Cluster UUID : {cluster_uuid}") + + cluster_secret = sbctl_raw("cluster", "get-secret", cluster_uuid) + if not cluster_secret: + print("ERROR: could not retrieve cluster secret.", file=sys.stderr) + sys.exit(1) + print(f" Secret : {'*' * min(len(cluster_secret), 8)}… (len={len(cluster_secret)})") + + # ── 3. Management-node IP ──────────────────────────────────────────────── + + print("\n[2] Resolving management node …") + if args.mgmt_ip: + mgmt_ip = args.mgmt_ip + print(f" Using provided IP : {mgmt_ip}") + else: + cp_nodes = sbctl_json("control-plane", "list") + if not cp_nodes: + print("ERROR: 'sbctl control-plane list' returned nothing.", file=sys.stderr) + sys.exit(1) + mgmt_ip = cp_nodes[0]["IP"] + print(f" Management IP : {mgmt_ip} ({len(cp_nodes)} node(s) total)") + + if args.mode == "kubernetes": + graylog_base = f"http://{mgmt_ip}:9000/api" + opensearch_base = f"http://{mgmt_ip}:9200" + else: + graylog_base = f"http://{mgmt_ip}/graylog/api" + opensearch_base = f"http://{mgmt_ip}/opensearch" + + # ── 4. Storage nodes ───────────────────────────────────────────────────── + + print("\n[3] Retrieving storage nodes …") + sn_list = sbctl_json("storage-node", "list") or [] + if not sn_list: + print(" WARN: no storage nodes found (continuing without them).") + else: + print(f" Found {len(sn_list)} storage node(s).") + + # ── 5. HTTP sessions ───────────────────────────────────────────────────── + + graylog_password = args.monitoring_secret if args.monitoring_secret else cluster_secret + if args.monitoring_secret: + print(" Using provided --monitoring-secret for Graylog auth.") + + gl_session = requests.Session() + gl_session.auth = ("admin", graylog_password) + gl_session.headers.update({"X-Requested-By": "sb-log-collector"}) + + os_session = requests.Session() + + # Verify Graylog reachability (informational only) + if not args.use_opensearch: + print(f"\n[4] Checking Graylog at {graylog_base} …") + try: + r = gl_session.get(f"{graylog_base}/system", timeout=10) + if r.status_code == 200: + ver = r.json().get("version", "?") + print(f" OK (version {ver})") + else: + print(f" WARN: HTTP {r.status_code} – will still attempt collection.") + except requests.RequestException as exc: + print(f" WARN: {exc} – will still attempt collection.") + else: + print(f"\n[4] Checking OpenSearch at {opensearch_base} …") + try: + r = os_session.get(f"{opensearch_base}/_cluster/health", timeout=10) + if r.status_code == 200: + status = r.json().get("status", "?") + print(f" OK (cluster status: {status})") + else: + print(f" WARN: HTTP {r.status_code}.") + except requests.RequestException as exc: + print(f" WARN: {exc}.") + + # --diagnose: print full report and exit + if args.diagnose: + opensearch_diagnose(os_session, opensearch_base, from_iso, to_iso) + sys.exit(0) + + # ── 6. Prepare temp workspace ──────────────────────────────────────────── + + ts_str = start_dt.strftime("%Y%m%d_%H%M%S") + bundle_name = f"sb_logs_{ts_str}_{args.duration_minutes}m" + output_dir = Path(args.output_dir).resolve() + output_dir.mkdir(parents=True, exist_ok=True) + tarball_path = output_dir / f"{bundle_name}.tar.gz" + + probe_cache: dict = {} # shared across all OpenSearch calls in this run + + fetch_kw = dict( + gl_session=gl_session, + os_session=os_session, + graylog_base=graylog_base, + opensearch_base=opensearch_base, + use_opensearch=args.use_opensearch, + from_iso=from_iso, + to_iso=to_iso, + probe_cache=probe_cache, + ) + + with tempfile.TemporaryDirectory() as tmpdir: + log_root = Path(tmpdir) / bundle_name + log_root.mkdir() + + # ── 7. Control-plane logs ──────────────────────────────────────────── + + cp_services = ( + CONTROL_PLANE_SERVICES_KUBERNETES + if args.mode == "kubernetes" + else CONTROL_PLANE_SERVICES_DOCKER + ) + print(f"\n[5] Collecting control-plane logs ({len(cp_services)} services, mode={args.mode}) …") + cp_dir = log_root / "control_plane" + cp_dir.mkdir() + + gl_cname_field = "kubernetes_container_name" if args.mode == "kubernetes" else "container_name" + + total_cp_lines = 0 + for svc in cp_services: + out_f = cp_dir / f"{svc}.log" + gl_q = f'{gl_cname_field}:/.*{_gl_escape(svc)}.*/' + n = fetch( + gl_query=gl_q, + os_container=svc, + os_source=None, + out_path=out_f, + **fetch_kw, + ) + total_cp_lines += n + status = f"{n:>8,} lines" + print(f" {svc:<42} {status}") + + print(f" {'Control-plane total':<42} {total_cp_lines:>8,} lines") + + # Every control-plane source empty means the log pipeline is not + # delivering, not that nothing happened. Say so loudly and fall back to + # the Docker service logs, otherwise the run ends with "Done!" and a + # tarball full of 0-byte files (2026-08-10: an entire fail-over + # investigation had no service logs because of exactly this). + if total_cp_lines == 0: + print("\n !! WARNING: 0 control-plane log lines retrieved.") + print(" !! Graylog/OpenSearch is not returning data for this window.") + if args.mode == "docker": + _collect_docker_service_logs(cp_dir) + else: + print(" !! Check the log pipeline before trusting this bundle.") + + # ── 8. Storage-node logs ───────────────────────────────────────────── + # Docker mode: collect SPDK/SNodeAPI logs from Graylog/OpenSearch. + # Kubernetes mode: SPDK logs are captured via kubectl in step 9. + + if args.mode == "docker": + print("\n[6] Collecting storage-node logs (docker) …") + sn_root = log_root / "storage_nodes" + sn_root.mkdir() + + # SNodeAPI runs on every storage node under the same container name. + # Its GELF 'source' field is the Docker host hostname whose exact + # format varies by deployment and cannot be reliably derived from + # the management IP alone. Collect ALL SNodeAPI logs once (no + # source filter) into a shared file; each line contains src= + # so per-node filtering can be done with grep afterwards. + print("\n SNodeAPI (all nodes combined) …") + snode_api_log = sn_root / "SNodeAPI_all_nodes.log" + snode_api_count = fetch( + gl_query='container_name:"SNodeAPI"', + os_container="SNodeAPI", + os_source=None, + out_path=snode_api_log, + **fetch_kw, + ) + print(f" {'SNodeAPI (all nodes)':<42} {snode_api_count:>8,} lines") + print(" (filter by src= to isolate per-node logs)") + + for node in sn_list: + hostname = node.get("Hostname", "unknown") + node_ip = node.get("Management IP", "") + rpc_port = node.get("SPDK P", 8080) + + node_label = f"{hostname}_{node_ip}".strip("_") if node_ip else hostname + node_dir = sn_root / node_label + node_dir.mkdir() + + print(f"\n Node: {hostname} ip={node_ip} rpc_port={rpc_port}") + + # spdk_N and spdk_proxy_N are globally unique by RPC port number; + # no source filter needed. + spdk_containers = [ + (f"spdk_{rpc_port}", f"spdk_{rpc_port}.log"), + (f"spdk_proxy_{rpc_port}", f"spdk_proxy_{rpc_port}.log"), + ] + + for cname, fname in spdk_containers: + out_f = node_dir / fname + n = fetch( + gl_query=f'container_name:"{cname}"', + os_container=cname, + os_source=None, + out_path=out_f, + **fetch_kw, + ) + print(f" {cname:<42} {n:>8,} lines") + else: + print("\n[6] Collecting storage-node logs (kubernetes) …") + sn_root = log_root / "storage_nodes" + sn_root.mkdir() + + for node in sn_list: + hostname = node.get("Hostname", "unknown") + node_ip = node.get("Management IP", "") + rpc_port = node.get("SPDK P", 8080) + + node_label = f"{hostname}_{node_ip}".strip("_") if node_ip else hostname + node_dir = sn_root / node_label + node_dir.mkdir() + + print(f"\n Node: {hostname} ip={node_ip} rpc_port={rpc_port}") + + # Pod name pattern: snode-spdk-pod-- + # Container names inside that pod: spdk-container, spdk-proxy-container + pod_name = f"snode-spdk-pod-{rpc_port}-*" + spdk_containers = [ + ("spdk-container", f"spdk-container_{rpc_port}.log"), + ("spdk-proxy-container", f"spdk-proxy-container_{rpc_port}.log"), + ] + + for cname, fname in spdk_containers: + out_f = node_dir / fname + gl_q = ( + f'kubernetes_pod_name:{_gl_escape(pod_name)} ' + f'AND kubernetes_container_name:/.*{_gl_escape(cname)}.*/' + ) + n = fetch( + gl_query=gl_q, + os_container=cname, + os_source=None, + os_pod_name=pod_name, + out_path=out_f, + **fetch_kw, + ) + print(f" {cname:<42} {n:>8,} lines") + + # ── 9. Kubernetes pod logs (CSI node + storage-node DS) ────────────── + + k8s_ns = args.namespace + if k8s_ns: + print(f"\n[7] Collecting Kubernetes pod logs (namespace: {k8s_ns}) …") + k8s_dir = log_root / "k8s_pods" + k8s_dir.mkdir() + + # 9a. simplyblock-csi-node* pods — all containers + dmesg + csi_pods = _kubectl_list_pods(k8s_ns, "simplyblock-csi-node") + if csi_pods: + csi_dir = k8s_dir / "csi-node" + csi_dir.mkdir() + print(f" CSI node pods ({len(csi_pods)}) …") + for pod in csi_pods: + collect_k8s_pod_logs(k8s_ns, pod, csi_dir, from_iso, to_iso) + collect_k8s_csi_dmesg(k8s_ns, pod, csi_dir, from_iso, to_iso) + else: + print(f" No simplyblock-csi-node pods found in namespace {k8s_ns}.") + + # 9b. simplyblock-csi-controller* pods — all containers + csi_ctrl_pods = _kubectl_list_pods(k8s_ns, "simplyblock-csi-controller") + if csi_ctrl_pods: + csi_ctrl_dir = k8s_dir / "csi-controller" + csi_ctrl_dir.mkdir() + print(f" CSI controller pods ({len(csi_ctrl_pods)}) …") + for pod in csi_ctrl_pods: + collect_k8s_pod_logs(k8s_ns, pod, csi_ctrl_dir, from_iso, to_iso) + else: + print(f" No simplyblock-csi-controller pods found in namespace {k8s_ns}.") + + # 9c. simplyblock-manager* pods — all containers + mgr_pods = _kubectl_list_pods(k8s_ns, "simplyblock-manager") + if mgr_pods: + mgr_dir = k8s_dir / "simplyblock-manager" + mgr_dir.mkdir() + print(f" Simplyblock manager pods ({len(mgr_pods)}) …") + for pod in mgr_pods: + collect_k8s_pod_logs(k8s_ns, pod, mgr_dir, from_iso, to_iso) + else: + print(f" No simplyblock-manager pods found in namespace {k8s_ns}.") + + # 9d. simplyblock-storage-node-ds* pods — all containers + sn_ds_pods = _kubectl_list_pods(k8s_ns, "simplyblock-storage-node-ds") + if sn_ds_pods: + sn_ds_dir = k8s_dir / "storage-node-ds" + sn_ds_dir.mkdir() + print(f" Storage-node DS pods ({len(sn_ds_pods)}) …") + for pod in sn_ds_pods: + collect_k8s_pod_logs(k8s_ns, pod, sn_ds_dir, from_iso, to_iso) + else: + print(f" No simplyblock-storage-node-ds pods found in namespace {k8s_ns}.") + else: + print("\n[7] Skipping Kubernetes pod logs (--namespace not set).") + + # ── 10. sbctl cluster / node snapshots ─────────────────────────────── + + print("\n[8] Collecting sbctl cluster / node info …") + info_dir = log_root / "sbctl_info" + info_dir.mkdir() + + def save_sbctl(label, cmd_args, out_name, use_json=False): + """Run sbctl, save output to out_name, print status.""" + if use_json: + data = sbctl_json(*cmd_args) + if data is not None: + out_path = info_dir / out_name + with open(out_path, "w") as f: + json.dump(data, f, indent=2) + print(f" {label:<50} OK ({out_name})") + return True + else: + text = sbctl_raw(*cmd_args) + if text is not None: + out_path = info_dir / out_name + out_path.write_text(text) + print(f" {label:<50} OK ({out_name})") + return True + print(f" {label:<50} FAILED", file=sys.stderr) + return False + + # 1. cluster show + save_sbctl( + "sbctl cluster show", + ["cluster", "show", cluster_uuid], + "cluster_show.txt", + ) + + # 2. lvol list + save_sbctl( + "sbctl lvol list", + ["lvol", "list", "--cluster-id", cluster_uuid], + "lvol_list.json", + use_json=True, + ) + + # 3. sn list (already fetched; save the raw JSON for completeness) + save_sbctl( + "sbctl sn list", + ["sn", "list"], + "sn_list.json", + use_json=True, + ) + + # 4. sn check – one file per storage node + print(" sbctl sn check (per node) …") + sn_check_dir = info_dir / "sn_check" + sn_check_dir.mkdir() + for node in sn_list: + node_uuid = node.get("UUID", "") + node_hostname = node.get("Hostname", node_uuid) + node_ip = node.get("Management IP", "") + label = f"{node_hostname}_{node_ip}".strip("_") if node_ip else node_hostname + text = sbctl_raw("sn", "check", node_uuid) + if text is not None: + (sn_check_dir / f"{label}.txt").write_text(text) + print(f" {label}") + else: + print(f" {label} FAILED", file=sys.stderr) + + # 5. cluster get-logs --limit 0 (all cluster-level events) + save_sbctl( + "sbctl cluster get-logs --limit 0", + ["cluster", "get-logs", cluster_uuid, "--limit", "0"], + "cluster_get_logs.txt", + ) + + # ── 11. Write a collection manifest ────────────────────────────────── + + manifest = { + "collected_at": datetime.now(timezone.utc).isoformat(), + "window_from": from_iso, + "window_to": to_iso, + "duration_minutes": args.duration_minutes, + "cluster_uuid": cluster_uuid, + "mgmt_ip": mgmt_ip, + "deploy_mode": args.mode, + "log_source": "opensearch-direct" if args.use_opensearch else "graylog-api", + "storage_nodes": [ + { + "hostname": n.get("Hostname"), + "ip": n.get("Management IP"), + "rpc_port": n.get("SPDK P"), + "uuid": n.get("UUID"), + } + for n in sn_list + ], + } + with open(log_root / "manifest.json", "w") as mf: + json.dump(manifest, mf, indent=2) + + # ── 12. Pack into tarball ───────────────────────────────────────────── + + print("\n[9] Creating tarball …") + with tarfile.open(str(tarball_path), "w:gz") as tar: + tar.add(str(log_root), arcname=bundle_name) + + size_mb = tarball_path.stat().st_size / 1_048_576 + print(f"\n{'=' * 64}") + print(" Done!") + print(f" Tarball : {tarball_path}") + print(f" Size : {size_mb:.2f} MB") + print(f"{'=' * 64}\n") + + +if __name__ == "__main__": + main() diff --git a/simplyblock_core/scripts/collect_logs.py b/simplyblock_core/scripts/collect_logs.py index 90436e5d91..cff784c93d 100755 --- a/simplyblock_core/scripts/collect_logs.py +++ b/simplyblock_core/scripts/collect_logs.py @@ -117,7 +117,7 @@ "TasksRunnerJCCompResume", "TasksRunnerLVolSyncDelete", "TasksRunnerBackup", - "TasksRunnerBackupMerge", + "BackupMergeService", "HAProxy", ] diff --git a/simplyblock_core/scripts/docker-compose-swarm.yml b/simplyblock_core/scripts/docker-compose-swarm.yml index f31bce6687..477d46ef7d 100644 --- a/simplyblock_core/scripts/docker-compose-swarm.yml +++ b/simplyblock_core/scripts/docker-compose-swarm.yml @@ -582,10 +582,10 @@ services: environment: SIMPLYBLOCK_LOG_LEVEL: "$LOG_LEVEL" - TasksRunnerBackupMerge: + BackupMergeService: <<: *service-base image: $SIMPLYBLOCK_DOCKER_IMAGE - command: "python3 simplyblock_core/services/tasks_runner_backup_merge.py" + command: "python3 simplyblock_core/services/backup_merge_service.py" deploy: placement: constraints: [node.role == manager] diff --git a/simplyblock_core/services/tasks_runner_backup_merge.py b/simplyblock_core/services/backup_merge_service.py similarity index 83% rename from simplyblock_core/services/tasks_runner_backup_merge.py rename to simplyblock_core/services/backup_merge_service.py index d28cfdc9a8..74b8976671 100644 --- a/simplyblock_core/services/tasks_runner_backup_merge.py +++ b/simplyblock_core/services/backup_merge_service.py @@ -1,6 +1,6 @@ # coding=utf-8 """ -tasks_runner_backup_merge.py - periodic service that evaluates backup policies +backup_merge_service.py - periodic service that evaluates backup policies and triggers merges when retention limits are exceeded. """ import time @@ -16,12 +16,6 @@ def main(): logger.info("Starting backup merge service...") while True: - try: - db.get_clusters() - except Exception as e: - logger.error(f"Failed to get clusters: {e}") - time.sleep(3) - continue clusters = db.get_clusters() for cl in clusters: if cl.status == Cluster.STATUS_IN_ACTIVATION: From b77632d616d33542f0f65596c184c9983cbb0b09 Mon Sep 17 00:00:00 2001 From: Max Schettler Date: Sun, 26 Jul 2026 22:21:40 +0200 Subject: [PATCH 05/25] Introduce generic task runner --- simplyblock_core/services/task_runner_base.py | 287 ++++++++++++++++++ tests/unit/tasks/test_task_runner_base.py | 242 +++++++++++++++ 2 files changed, 529 insertions(+) create mode 100644 simplyblock_core/services/task_runner_base.py create mode 100644 tests/unit/tasks/test_task_runner_base.py diff --git a/simplyblock_core/services/task_runner_base.py b/simplyblock_core/services/task_runner_base.py new file mode 100644 index 0000000000..9c923523e9 --- /dev/null +++ b/simplyblock_core/services/task_runner_base.py @@ -0,0 +1,287 @@ +# coding=utf-8 +"""Shared driver for the task runners. + +A task runner is a long-lived service that polls FoundationDB for `JobSchedule` +tasks of one or more function names and advances each one. Historically every +runner hand-rolled its own ``while True`` loop, lease handling, retry ceiling and +error plumbing, which drifted apart. This module centralizes that skeleton so a +runner is reduced to a :class:`RunnerSpec` — most importantly a *handler* that +does only the domain work. + +Handler contract +---------------- +The handler is a callable ``handler(task) -> None``. It performs its domain work +(including mutating its own domain models — Backup, LVol, migration, … — and +writing those) but it MUST NOT touch task lifecycle state (``status`` / +``retry``) or call ``task.write_to_db`` for the task: the driver owns all of +that. The handler signals its outcome purely through ordinary Python control +flow: + +- **return** (``None``) — the task is terminally complete → ``STATUS_DONE``. +- **raise** :class:`TaskDefer` — not terminal: still in progress, or blocked on + external state. Re-poll next cycle; **no retry consumed**; no backoff. +- **raise** :class:`TaskRetry` (or any other, unexpected ``Exception``) — a + retryable failure. Suspend, **consume a retry**, and back off before the next + attempt. +- **raise** :class:`TaskAbort` — a permanent, non-retryable stop (missing param, + object gone, "not needed"). Finish the task (``STATUS_DONE``) with the reason. + +DB errors are deliberately NOT caught: an unhandled ``get_clusters`` / +``get_job_tasks`` failure propagates out of :func:`serve`, exits the process +non-zero, and lets the orchestrator restart it with a fresh FDB connection. +""" +import threading +import time +from concurrent.futures import ThreadPoolExecutor +from dataclasses import dataclass +from typing import Any, Callable, Optional, Sequence + +from simplyblock_core import constants, db_controller, utils +from simplyblock_core.controllers import tasks_controller +from simplyblock_core.models.job_schedule import JobSchedule + +logger = utils.get_logger(__name__) + +db = db_controller.DBController() + +# Cap the per-task exponential backoff so a permanently-failing task can't grow +# its retry delay without bound. +_BACKOFF_CAP_SEC = constants.RESTART_TASK_EXEC_INTERVAL_MAX_SEC + + +class TaskDefer(Exception): + """Handler signal: the task is not terminal — still in progress, or blocked + on external state. Re-poll next cycle without consuming a retry.""" + + +class TaskRetry(Exception): + """Handler signal: a retryable failure. Suspend, consume a retry, back off. + + Any other unexpected ``Exception`` from a handler is treated identically.""" + + +class TaskAbort(Exception): + """Handler signal: a permanent, non-retryable stop. Finish the task.""" + + +def _default_eligible(task: JobSchedule, cluster: Any) -> bool: + return True + + +@dataclass +class RunnerSpec: + """Describes one task runner. ``function_names`` and ``handler`` are the only + required fields; the rest default to a simple serial runner.""" + + function_names: Sequence[str] + handler: Callable[[JobSchedule], None] + name: str = "task-runner" + # Pure, side-effect-free "can I run this task right now?" predicate. The + # default always-eligible keeps simple runners trivial. A task judged + # ineligible is skipped this cycle without a lease claim or a write, exactly + # like the ad-hoc IN_ACTIVATION / same-node-sibling gates it replaces. + is_eligible: Callable[[JobSchedule, Any], bool] = _default_eligible + interval: float = constants.TASK_EXEC_INTERVAL_SEC + # Serial by default. > 1 runs tasks on a thread pool of this size. + concurrency: int = 1 + # Optional per-key mutual exclusion for concurrent mode: two tasks whose + # exclusion_key() is equal never run at the same time (e.g. one restart per + # node). Ignored when concurrency == 1. + exclusion_key: Optional[Callable[[JobSchedule], Any]] = None + + def __post_init__(self) -> None: + if self.concurrency < 1: + raise ValueError("concurrency must be >= 1") + + +class TaskRunner: + """Drives the tasks matched by a :class:`RunnerSpec`. See module docstring + for the handler contract.""" + + def __init__(self, spec: RunnerSpec): + self.spec = spec + self._executor = ( + ThreadPoolExecutor(max_workers=spec.concurrency, + thread_name_prefix=spec.name) + if spec.concurrency > 1 else None) + self._lock = threading.Lock() + # task uuid -> Future-in-flight guard (this host), so the dispatch loop + # never hands the same task to two workers. Cross-host duplicate + # execution is prevented separately by the per-task lease. + self._inflight: set = set() + self._inflight_keys: dict = {} # exclusion key -> task uuid + self._next_attempt: dict = {} # task uuid -> earliest retry timestamp + + # -- public entrypoint -------------------------------------------------- + + def run(self) -> None: + logger.info(f"Starting {self.spec.name}...") + while True: + # DB errors are intentionally uncaught: they propagate out, exit the + # process, and the orchestrator restarts us with a fresh FDB client. + clusters = db.get_clusters() + if not clusters: + logger.error("No clusters found!") + else: + for cl in clusters: + for task in db.get_job_tasks(cl.get_id(), reverse=False): + if task.function_name not in self.spec.function_names: + continue + if task.status == JobSchedule.STATUS_DONE: + self._forget(task.uuid) + continue + self._dispatch(task, cl) + time.sleep(self.spec.interval) + + # -- dispatch ----------------------------------------------------------- + + def _dispatch(self, task: JobSchedule, cluster: Any) -> None: + uuid = task.uuid + # Backoff gate: a task not yet due is skipped so a waiting task does not + # block the others behind it (the loop revisits every task each cycle). + if time.time() < self._next_attempt.get(uuid, 0): + return + + if self._executor is None: + self._process(task, cluster) + return + + with self._lock: + if uuid in self._inflight: + return + key = self.spec.exclusion_key(task) if self.spec.exclusion_key else None + if key is not None and key in self._inflight_keys: + return + self._inflight.add(uuid) + if key is not None: + self._inflight_keys[key] = uuid + self._executor.submit(self._process_worker, task, cluster) + + def _process_worker(self, task: JobSchedule, cluster: Any) -> None: + # A worker crash must be contained to this task, never kill the service + # loop or leave the task wedged in the in-flight set. + try: + self._process(task, cluster) + except Exception as e: # noqa: BLE001 - contain crash to this worker + logger.error(f"{self.spec.name}: task {task.uuid} crashed in worker: {e}") + logger.exception(e) + finally: + self._release_inflight(task.uuid) + + # -- per-task lifecycle ------------------------------------------------- + + def _process(self, task: JobSchedule, cluster: Any) -> None: + uuid = task.uuid + + # Pre-run skip-gate 1 — eligibility (pure, no write): not ready yet. + if not self.spec.is_eligible(task, cluster): + return + + # Pre-run skip-gate 2 — lease: another live host owns this task. + if not tasks_controller.claim_task(task): + logger.info(f"{self.spec.name}: task {uuid} owned by another runner host; skipping") + return + + # Authoritative re-fetch AFTER the claim: claim_task mutated the DB copy + # (owner / updated_at) but not this local object, and write_to_db writes + # the whole object — reading the fresh, lease-stamped copy here is what + # keeps a later write from clobbering the owner. + task = db.get_task_by_id(uuid) + if task is None or task.status == JobSchedule.STATUS_DONE: + self._forget(uuid) + return + + if task.canceled: + self._finish(task, "canceled") + return + if 0 <= task.max_retry <= task.retry: + self._finish(task, "max retry reached") + return + + if task.status != JobSchedule.STATUS_RUNNING: + task.status = JobSchedule.STATUS_RUNNING + task.write_to_db(db.kv_store) + + try: + # Heartbeat the lease for the duration of the handler: TASK_LEASE_TTL + # is far shorter than a node-add / restart / migration, so a lease + # refreshed only on task writes would go stale mid-handler and let a + # second host claim and double-drive the task. + with tasks_controller.task_lease_heartbeat(task): + self.spec.handler(task) + except TaskDefer as e: + self._defer(task, str(e)) + except TaskAbort as e: + self._finish(task, str(e) or "aborted") + except TaskRetry as e: + self._fail(task, str(e) or "retry") + except Exception as e: # noqa: BLE001 - unexpected == retryable failure + logger.error(f"{self.spec.name}: task {uuid} handler raised: {e}") + logger.exception(e) + self._fail(task, f"unhandled error: {e}") + else: + self._succeed(task) + + # -- outcome transitions (the only places task state is mutated) -------- + + def _succeed(self, task: JobSchedule) -> None: + task.status = JobSchedule.STATUS_DONE + if not task.function_result: + task.function_result = "completed" + task.write_to_db(db.kv_store) + self._forget(task.uuid) + + def _finish(self, task: JobSchedule, result: str) -> None: + """Terminal DONE for a non-handler-success reason (canceled, max retry, + abort).""" + task.function_result = result + task.status = JobSchedule.STATUS_DONE + task.write_to_db(db.kv_store) + self._forget(task.uuid) + + def _defer(self, task: JobSchedule, reason: str) -> None: + if reason: + task.function_result = reason + task.status = JobSchedule.STATUS_SUSPENDED + task.write_to_db(db.kv_store) + self._clear_backoff(task.uuid) + + def _fail(self, task: JobSchedule, reason: str) -> None: + task.retry += 1 + task.function_result = reason + task.status = JobSchedule.STATUS_SUSPENDED + task.write_to_db(db.kv_store) + with self._lock: + self._next_attempt[task.uuid] = time.time() + self._backoff_delay(task.retry) + + # -- bookkeeping -------------------------------------------------------- + + def _backoff_delay(self, retry: int) -> float: + if retry <= 0: + return 0.0 + exp = min(retry - 1, 16) # guard the shift against absurd retry counts + return min(self.spec.interval * (2 ** exp), _BACKOFF_CAP_SEC) + + def _clear_backoff(self, uuid: str) -> None: + with self._lock: + self._next_attempt.pop(uuid, None) + + def _release_inflight(self, uuid: str) -> None: + with self._lock: + self._inflight.discard(uuid) + for key, owner_uuid in list(self._inflight_keys.items()): + if owner_uuid == uuid: + del self._inflight_keys[key] + + def _forget(self, uuid: str) -> None: + with self._lock: + self._next_attempt.pop(uuid, None) + self._inflight.discard(uuid) + for key, owner_uuid in list(self._inflight_keys.items()): + if owner_uuid == uuid: + del self._inflight_keys[key] + + +def serve(spec: RunnerSpec) -> None: + """Instantiate and run the driver for ``spec`` (a runner's ``main``).""" + TaskRunner(spec).run() diff --git a/tests/unit/tasks/test_task_runner_base.py b/tests/unit/tasks/test_task_runner_base.py new file mode 100644 index 0000000000..ffe556cae8 --- /dev/null +++ b/tests/unit/tasks/test_task_runner_base.py @@ -0,0 +1,242 @@ +# coding=utf-8 +"""Unit tests for the shared task-runner driver (``task_runner_base``). + +These exercise the per-task lifecycle (``_process``) and the dispatch loop +(``run``) directly, with ``db`` and ``claim_task`` mocked. They pin the handler +contract: return → DONE, TaskDefer → suspend without a retry, TaskRetry / +unexpected Exception → suspend + retry + backoff, TaskAbort → DONE; the pre-run +skip-gates (eligibility, lease); and that a handler exception never escapes the +loop while a DB error does. +""" +import time +from unittest.mock import MagicMock + +import pytest + +from simplyblock_core.models.job_schedule import JobSchedule +import simplyblock_core.services.task_runner_base as trb + + +@pytest.fixture(autouse=True) +def _no_task_writes(monkeypatch): + """Task writes would dereference a None kv_store; the driver's state + transitions are asserted on the in-memory task object instead.""" + monkeypatch.setattr(JobSchedule, "write_to_db", MagicMock()) + + +def _task(status=JobSchedule.STATUS_NEW, retry=0, max_retry=8, canceled=False): + task = JobSchedule() + task.uuid = "task-1" + task.function_name = "fn" + task.cluster_id = "cl-1" + task.node_id = "node-1" + task.status = status + task.retry = retry + task.max_retry = max_retry + task.canceled = canceled + task.function_params = {} + return task + + +def _wire(monkeypatch, task, claim=True): + """Mock the module ``db`` (re-fetch returns the same task), the lease, and + the lease heartbeat (a no-op context manager so no real thread is spawned).""" + db = MagicMock() + db.get_task_by_id.return_value = task + monkeypatch.setattr(trb, "db", db) + monkeypatch.setattr(trb.tasks_controller, "claim_task", lambda *a, **k: claim) + monkeypatch.setattr(trb.tasks_controller, "task_lease_heartbeat", MagicMock()) + return db + + +def _runner(handler, **spec_kw): + spec = trb.RunnerSpec(function_names=("fn",), handler=handler, **spec_kw) + return trb.TaskRunner(spec) + + +# -- handler outcome vocabulary -------------------------------------------- + +def test_void_return_marks_done(monkeypatch): + task = _task() + _wire(monkeypatch, task) + handler = MagicMock(return_value=None) + _runner(handler)._process(task, MagicMock()) + handler.assert_called_once_with(task) + assert task.status == JobSchedule.STATUS_DONE + + +def test_handler_runs_under_lease_heartbeat(monkeypatch): + task = _task() + _wire(monkeypatch, task) + hb = MagicMock() + monkeypatch.setattr(trb.tasks_controller, "task_lease_heartbeat", hb) + _runner(MagicMock(return_value=None))._process(task, MagicMock()) + hb.assert_called_once_with(task) + hb.return_value.__enter__.assert_called_once() + hb.return_value.__exit__.assert_called_once() + + +def test_defer_suspends_without_consuming_retry(monkeypatch): + task = _task(retry=2) + _wire(monkeypatch, task) + + def handler(_task): + raise trb.TaskDefer("node not online") + + runner = _runner(handler) + runner._process(task, MagicMock()) + assert task.status == JobSchedule.STATUS_SUSPENDED + assert task.retry == 2 + assert task.function_result == "node not online" + assert "task-1" not in runner._next_attempt # no backoff for a defer + + +def test_retry_suspends_consumes_retry_and_backs_off(monkeypatch): + task = _task(retry=1) + _wire(monkeypatch, task) + + def handler(_task): + raise trb.TaskRetry("rpc failed") + + runner = _runner(handler) + runner._process(task, MagicMock()) + assert task.status == JobSchedule.STATUS_SUSPENDED + assert task.retry == 2 + assert runner._next_attempt["task-1"] > time.time() + + +def test_unexpected_exception_is_treated_as_retry(monkeypatch): + task = _task(retry=0) + _wire(monkeypatch, task) + + def handler(_task): + raise RuntimeError("boom") + + runner = _runner(handler) + runner._process(task, MagicMock()) # must not raise + assert task.status == JobSchedule.STATUS_SUSPENDED + assert task.retry == 1 + + +def test_abort_marks_done_with_reason(monkeypatch): + task = _task() + _wire(monkeypatch, task) + + def handler(_task): + raise trb.TaskAbort("missing param") + + _runner(handler)._process(task, MagicMock()) + assert task.status == JobSchedule.STATUS_DONE + assert task.function_result == "missing param" + + +# -- pre-run skip-gates ----------------------------------------------------- + +def test_ineligible_skips_without_claim_or_write(monkeypatch): + task = _task() + _wire(monkeypatch, task) + claim = MagicMock(return_value=True) + monkeypatch.setattr(trb.tasks_controller, "claim_task", claim) + handler = MagicMock() + + _runner(handler, is_eligible=lambda t, c: False)._process(task, MagicMock()) + + handler.assert_not_called() + claim.assert_not_called() + JobSchedule.write_to_db.assert_not_called() + assert task.status == JobSchedule.STATUS_NEW + + +def test_default_eligible_runs(monkeypatch): + task = _task() + _wire(monkeypatch, task) + handler = MagicMock(return_value=None) + _runner(handler)._process(task, MagicMock()) + handler.assert_called_once() + + +def test_lease_denied_skips(monkeypatch): + task = _task() + _wire(monkeypatch, task, claim=False) + handler = MagicMock() + _runner(handler)._process(task, MagicMock()) + handler.assert_not_called() + assert task.status == JobSchedule.STATUS_NEW + + +# -- terminal pre-handler checks ------------------------------------------- + +def test_canceled_marks_done_without_handler(monkeypatch): + task = _task(canceled=True) + _wire(monkeypatch, task) + handler = MagicMock() + _runner(handler)._process(task, MagicMock()) + handler.assert_not_called() + assert task.status == JobSchedule.STATUS_DONE + assert task.function_result == "canceled" + + +def test_max_retry_marks_done_without_handler(monkeypatch): + task = _task(retry=8, max_retry=8) + _wire(monkeypatch, task) + handler = MagicMock() + _runner(handler)._process(task, MagicMock()) + handler.assert_not_called() + assert task.status == JobSchedule.STATUS_DONE + assert "max retry" in task.function_result + + +def test_negative_max_retry_is_unbounded(monkeypatch): + task = _task(retry=100, max_retry=-1) + _wire(monkeypatch, task) + handler = MagicMock(return_value=None) + _runner(handler)._process(task, MagicMock()) + handler.assert_called_once() # ceiling never binds for max_retry < 0 + assert task.status == JobSchedule.STATUS_DONE + + +# -- dispatch loop ---------------------------------------------------------- + +class _StopLoop(Exception): + pass + + +def test_run_dispatches_only_matching_non_done(monkeypatch): + match = _task() + other = _task() + other.uuid = "other" + other.function_name = "nope" + done = _task() + done.uuid = "done" + done.status = JobSchedule.STATUS_DONE + + cluster = MagicMock() + cluster.get_id.return_value = "cl-1" + db = MagicMock() + db.get_clusters.return_value = [cluster] + db.get_job_tasks.return_value = [match, other, done] + monkeypatch.setattr(trb, "db", db) + monkeypatch.setattr(trb.time, "sleep", MagicMock(side_effect=_StopLoop)) + + runner = _runner(MagicMock()) + dispatched = [] + monkeypatch.setattr(runner, "_dispatch", lambda t, c: dispatched.append(t.uuid)) + + with pytest.raises(_StopLoop): + runner.run() + assert dispatched == ["task-1"] + + +def test_run_propagates_db_error(monkeypatch): + db = MagicMock() + db.get_clusters.side_effect = RuntimeError("fdb down") + monkeypatch.setattr(trb, "db", db) + with pytest.raises(RuntimeError, match="fdb down"): + _runner(MagicMock()).run() + + +# -- spec validation -------------------------------------------------------- + +def test_spec_rejects_zero_concurrency(): + with pytest.raises(ValueError): + trb.RunnerSpec(function_names=("fn",), handler=MagicMock(), concurrency=0) From c6a003d6f8cf8e28ce8af5a3ea4a94db7d8b68e6 Mon Sep 17 00:00:00 2001 From: Max Schettler Date: Mon, 27 Jul 2026 14:00:47 +0200 Subject: [PATCH 06/25] Handover documents --- HANDOFF.md | 80 ++++++++++ TASK_RUNNER_REWORK_PLAN.md | 308 +++++++++++++++++++++++++++++++++++++ 2 files changed, 388 insertions(+) create mode 100644 HANDOFF.md create mode 100644 TASK_RUNNER_REWORK_PLAN.md diff --git a/HANDOFF.md b/HANDOFF.md new file mode 100644 index 0000000000..b56d0bf649 --- /dev/null +++ b/HANDOFF.md @@ -0,0 +1,80 @@ +# Task-runner rework — handoff + +Continuation notes for picking up the `rework-task-runners` branch on another +machine. The full design lives in **`TASK_RUNNER_REWORK_PLAN.md`** (same dir) — +read it first; this file is just the current state + how to continue. + +## Where the work is + +Branch `rework-task-runners`, rebased onto `origin/main`. Five commits: + +``` +6ea41859c Introduce generic task runner (B0: driver + lease heartbeat + tests) +8458a5e44 Reclassify backup policy evaluation… (A5: backup_merge -> backup_merge_service) +6079ed010 Apply task lease consistently… (A3: claim_task on the lease-less runners) +4882cecac Unify task retry semantics (A2: canonical 0 <= max_retry <= retry) +9da6cd5f7 Consistently crash task runners on DB… (A1: remove DB-error masking) +``` + +Pre-rebase safety backup: branch `backup/rework-pre-rebase2` (`61cd76770`). + +## Status: Phase A done, B0 done, Phase B (migrations) NOT started + +- **A1/A2/A3/A5** — done. **A4** was folded into Phase B (no commit — see plan). +- **B0** — `simplyblock_core/services/task_runner_base.py` (the `TaskRunner` + driver: loop, single post-claim re-fetch, eligibility + lease pre-run gates, + `task_lease_heartbeat` around the handler, retry ceiling + backoff, serial / + opt-in concurrency) + `tests/unit/tasks/test_task_runner_base.py`. **Done.** +- **B1…B9** — migrate each runner onto the driver, one commit each, in the order + listed in the plan (`fdb_backup, jc_comp, replication_final, sync_lvol_del` → + `backup, cluster_expand` → `node_add` → migration trio → `port_allow` → + `restart` → `lvol_migration` → `node_removal` → `batch_migration`). **Not started.** + +## Gotchas (read before continuing) + +- **`tests/unit/tasks/test_retry_ceiling.py` HANGS — and it hangs on `origin/main` + itself.** Upstream restructured the runners it drives (module-level loops, + `task_runner`→`process_task`) without updating it. Do NOT run the full + `tests/unit/tasks/` dir until it is skipped/removed. It exercises the old + `main()` loops that Phase B deletes; plan is to retire it as runners migrate. +- **Upstream already reworked the runners**: renamed `task_runner`→`process_task` + and added `tasks_controller.task_lease_heartbeat` to the six long-blocking + runners. B0's driver already folds heartbeat in, so migrating a runner should + DELETE its per-runner `process_task` + `claim_task` + `task_lease_heartbeat` + boilerplate in favor of `serve(SPEC)`. +- **Two new upstream runners** to migrate: `tasks_runner_node_removal.py` + (`FN_NODE_REMOVAL`) and `tasks_runner_batch_migration.py` (`FN_LVOL_BATCH_MIG`). +- Each migrated `main()` collapses to `serve(SPEC)`; each handler becomes **void** + and signals via `TaskDefer` / `TaskRetry` / `TaskAbort` (see plan §B0). Keep the + handler importable — tests call it directly. + +## Environment setup on the new machine + +```bash +git fetch +git checkout rework-task-runners +pip install -e . # editable install +# FoundationDB 7.3.3 client library (libfdb_c) must be installed on the host +``` + +## Verification (was NOT runnable on the origin machine — offline) + +The origin box had no network and a wiped `.tox`, so lint/types/tox could not run +there; the driver was verified with an ad-hoc stub harness only. **Re-run the real +checks on the new machine:** + +```bash +tox run-parallel -e lint,types +tox run -e unit -- tests/unit/tasks/test_task_runner_base.py \ + tests/unit/tasks/test_max_retry_semantics.py \ + tests/unit/test_task_lease.py \ + tests/unit/test_lvol_sync_op_task.py +# NOTE: do not add tests/unit/tasks/test_retry_ceiling.py — it hangs (see gotchas) +``` + +Follow the `tox-verify` skill / `AGENTS.md` for the full workflow. + +## Before opening the PR + +Delete these two handoff files (`HANDOFF.md`, `TASK_RUNNER_REWORK_PLAN.md`) — they +are transfer artifacts, not part of the change. diff --git a/TASK_RUNNER_REWORK_PLAN.md b/TASK_RUNNER_REWORK_PLAN.md new file mode 100644 index 0000000000..76a9218fa2 --- /dev/null +++ b/TASK_RUNNER_REWORK_PLAN.md @@ -0,0 +1,308 @@ +# Rework task runners into a resilient, concurrent, deduplicated framework + +## Context + +`simplyblock_core/services/` contains ~15 `tasks_runner_*.py` files. Each is a +hand-written `main()` `while True` loop plus a `task_runner()` that share the +same skeleton but diverge in almost every detail. The divergence is +**accidental** — the result of copy-paste plus independent incident fixes, not +deliberate per-runner design — and it produces real correctness gaps: + +- **DB-error handling is broken in most runners.** The dominant idiom calls + `db.get_clusters()` once inside a `try` (discarding the result), then calls it + **again unguarded** and uses *that* — so the guard is dead code and any + `FDBError` crashes the process (`tasks_runner_backup/migration/restart/` + `port_allow/lvol_migration/jc_comp/new_dev_migration/failed_migration`). + `tasks_runner_fdb_backup` has no guard at all. Per-cluster `get_job_tasks()` + is unguarded in almost all of them — this is the 2026-07-16 incident that + killed several runners with no auto-restart. Only `node_add` and + `sync_lvol_del` handle DB errors coherently. +- **`max_retry` semantics differ** (`retry >= max_retry` vs + `0 <= max_retry <= retry` vs `max_retry > 0 and retry >= max_retry`). With the + model default `max_retry = -1` (meant to be "unbounded"), the first form + terminates a task *immediately* (`0 >= -1`). Pinned intent lives in + `tests/unit/tasks/test_max_retry_semantics.py` and `test_retry_ceiling.py`. +- **The task lease (`tasks_controller.claim_task`) is applied in only 6 of the + runners.** The rest can double-execute side-effecting tasks during rolling + deploys / dual-manager windows. +- **Concurrency exists in only two runners** (`node_add`, `restart`: + `ThreadPoolExecutor` + per-task/per-node inflight sets + capped backoff); + `cluster_expand` has inline backoff; the rest are strictly serial. +- Sleep intervals, `IN_ACTIVATION` skipping, and task re-fetch placement are all + ad-hoc. + +**Deployment context:** runners are Docker Swarm services +(`simplyblock_core/scripts/docker-compose-swarm.yml`), which restart a container +on exit by default. So the intended way to survive a wedged FDB client is to +**let a DB error surface as a process failure (`sys.exit(1)`) and rely on Swarm +to restart with a fresh FDB connection.** + +**Intended outcome:** one shared driver that owns the loop, DB-error-to-exit, +task lease, standardized retry/lifecycle, and opt-in concurrency — so each +runner is reduced to its domain-specific "advance one task" handler. Reached via +small single-purpose commits so the branch stays easy to rebase onto upstream. + +### Decisions (confirmed with user) + +- **DB-error model: immediate `sys.exit(1)`.** No consecutive-failure threshold. + This *simplifies* the current `sync_lvol_del` threshold counter down to the + common model. +- **Scope: convert all task runners** onto the shared driver, with two + exceptions: + - `tasks_cluster_status.py` — a shell-command runner, not `FN_*`-task based; + being replaced by a deployment-level solution. **Leave untouched.** + - `tasks_runner_backup_merge.py` — a periodic *policy/schedule evaluator* over + lvols with no `FN_*` tasks. **Reclassify as a service**: rename to follow the + `*_service.py` / `*_monitor.py` convention already used in the directory + (`snapshot_monitor.py`, `lvol_monitor.py`, `health_check_service.py`, …). +- **Execution model: serial by default, concurrency opt-in** (per-task and + per-key exclusion + capped backoff), generalizing `node_add`/`restart`. +- **Exceptions are the failure-signaling mechanism.** Specific task functions + return `None` (void) and never touch task state; they *raise* to signal + failure, using ordinary Python function semantics. The driver owns **all** + `task.status` / `task.retry` / `write_to_db` mutation and translates the + handler's return-or-raise into the task's next state. Retryable vs + non-retryable is disambiguated by exception type (see the vocabulary in B0). + - **Placement pushback (accepted friction):** this cannot land in Phase A. A + handler can only stop mutating task state once a driver exists to own that + state — before then, a raised exception has nothing to catch it and would + crash the runner. So the contract is *defined* in **B0** and *realized* + per-runner during each **B1…Bn** migration. Phase A therefore leaves handler + internals alone on purpose (Phase B rewrites them), and A2/A4 only touch the + loop-level and terminate-decision code, not the handlers' error plumbing. + +### Constraints to preserve + +- Each runner keeps an **importable, unit-testable per-task entry point** + (`tests/unit/tasks/` imports the module and calls `task_runner(...)` directly; + this works because `main()` sits behind `if __name__ == "__main__"`). The + handler function must stay module-level and callable in isolation. +- Reuse existing helpers, do not reinvent: `tasks_controller.claim_task`, + `get_active_node_tasks`, `get_active_node_mig_task`, `is_auto_restart_paused`; + `constants.TASK_EXEC_INTERVAL_SEC`, `RESTART_TASK_EXEC_INTERVAL_MAX_SEC`, + `NODE_RESTART_MAX_PARALLEL_SUSPENDED`; `storage_node_ops.fd_dead_recovery_allowed`; + `snapshot_controller.lvstore_op_lock`. +- Keep documented deliberate exceptions (e.g. `port_allow` intentionally does + **not** skip `IN_ACTIVATION` clusters — preserve with its comment). + +--- + +## Phase A — Reconcile inconsistencies (no shared abstraction yet) + +Each item is one focused commit spanning the affected runners. These land first +so behavior is uniform *before* it is centralized, which also makes Phase B a +mechanical extraction. + +**A1 — DB errors become runner failures (immediate exit).** +In every converted runner's `main()`: delete the discard-then-recall +`get_clusters()` double-call; call it once and let exceptions propagate. Wrap the +loop body so any DB access failure (`get_clusters`, `get_job_tasks`, +`get_cluster_by_id`, …) logs and `sys.exit(1)`. Remove `sync_lvol_del`'s +`_DB_FAILURE_RESTART_THRESHOLD` counter (now redundant). Task-*handler* errors +stay caught and non-fatal — only infra/DB errors exit. +Files: all `tasks_runner_*` in scope. + +**A2 — Standardize `max_retry` semantics.** +One canonical rule: `max_retry < 0` ⇒ unbounded; otherwise terminate when +`retry >= max_retry`. Fix the runners using the immediate-terminate form. Verify +against `tests/unit/tasks/test_max_retry_semantics.py` and `test_retry_ceiling.py`. +Files: `node_add`, `jc_comp`, `fdb_backup`, `replication_final`, `backup`, +`cluster_expand`, migration trio, `restart`, `port_allow`, `lvol_migration`, +`sync_lvol_del`. + +**A3 — Apply the task lease uniformly.** +Add `tasks_controller.claim_task(task)` (skip-if-owned) to every side-effecting +runner that lacks it: `backup`, `fdb_backup`, `jc_comp`, `new_dev_migration`, +`failed_migration`, `replication_final`, `sync_lvol_del`. + +**A4 — Loop hygiene → folded into Phase B (no standalone commit).** +Investigation showed the three "hygiene" concerns are premature to reconcile in +Phase A and are owned correctly by the B0 driver + `RunnerSpec`: +- **Interval** — the literals deliberately differ (`3`/`5`/`10`/`60`s; + `TASK_EXEC_INTERVAL_SEC` is `10`), so unifying onto one constant changes polling + cadence. Each runner keeps its value via `spec.interval`. +- **`IN_ACTIVATION` skip** — task-dependent (some runners run during activation; + `port_allow` opts out by design). Folded into the **eligibility predicate** as + the condition `cluster.status != IN_ACTIVATION`, dropping the separate + `skip_in_activation` flag. +- **Re-fetch** — every runner *already* re-fetches before running; only the + *location* differs (main vs handler), which is harmless. The driver performs a + single consistent re-fetch before the handler; handlers stop re-fetching. + +**A5 — Reclassify `backup_merge` as a service.** +Rename `tasks_runner_backup_merge.py` → `backup_merge_service.py` (fix its own +dead double `get_clusters()` guard while moving it). Update the `command:` in +`docker-compose-swarm.yml` and any other references. It stays a plain periodic +service and does **not** move onto the driver in Phase B. + +--- + +## Phase B — Shared driver + per-runner migration + +**B0 — Introduce `simplyblock_core/services/task_runner_base.py`.** One commit, +new module + unit tests, no runner changed yet. Provides: + +- `serve(spec)` — the entrypoint wrapper: runs the loop; on any uncaught + DB/infra exception, logs and `sys.exit(1)` (the A1 model, centralized). +- The loop: `get_clusters()` → per-cluster `get_job_tasks()` → filter by the + spec's `function_names` and `status != DONE` → dispatch. All DB calls unguarded + (failures exit). (Activation-skipping is not a loop concern — it is an + eligibility condition, below.) +- Per-task lifecycle wrapper applied around the runner's handler, in order: + **single re-fetch** (`db.get_task_by_id`, the one authoritative fresh read — + handlers no longer re-fetch) → canceled→DONE → `max_retry` terminate → **pre-run + skip-gates** (below) → set `RUNNING` → **call the void handler** → map its + return-or-raise to the next task state and `write_to_db`. The wrapper is the + *only* place task state is mutated. Handler exceptions are caught here → task + updated per the vocabulary below, **loop survives**. +- **Pre-run skip-gates.** One category of gate, evaluated before the handler, + sharing one outcome: *skip this cycle, do not consume a retry, do not mutate task + lifecycle state*. Two members, applied in this order: + 1. **Eligibility** — `spec.is_eligible(task, cluster) -> bool`, **default + `True`**, a side-effect-free "can I run right now?" read. The default keeps + simple runners trivial; complex tasks supply a predicate over + cluster/node/sibling-task state. It is the abstraction for what the migration + family does ad-hoc today: `lvol_migration`'s pre-run chain (`src`/`tgt` node + online, `cluster.status in (ACTIVE, DEGRADED)`, no open + `get_active_cluster_expand_task`), the `get_active_node_mig_task` / + same-node-sibling exclusion in the migration trio, and `restart`'s + `is_auto_restart_paused`. **`IN_ACTIVATION` skipping is just an eligibility + condition** (`cluster.status != IN_ACTIVATION`): runners that pause during + activation include it in their predicate; `port_allow` (documented opt-out) + omits it — replacing the former separate `skip_in_activation` flag. Modeling + these as a *pre-run* gate rather than `TaskDefer` (an *in-handler* raise) + keeps the "just wait, never consume a retry" family unbounded exactly as + `test_retry_ceiling.py`'s `INTENTIONALLY_UNBOUNDED` set documents. + 2. **Lease** — the built-in `tasks_controller.claim_task(task)` gate, **always + applied** (no opt-out: it is universal double-execution protection). + Conceptually the lease *is* an eligibility question ("is this host eligible to + run this?") and shares the skip-without-retry outcome, so it lives in the same + category — but it stays a **distinct built-in gate, not folded into + `is_eligible`**, for three reasons: (a) a + successful claim has a **side-effect** (writes `owner`/`updated_at` to + acquire/refresh ownership) whereas `is_eligible` is a pure read; (b) it is + **cross-cutting** double-execution protection that a custom predicate must + never be able to silently drop; (c) it runs **after** the cheap pure + eligibility so an ineligible task short-circuits *before* the driver churns + the lease's `updated_at` on a task it will skip anyway. +- **Exception vocabulary** (defined here; handlers raise these, driver + interprets): + - *returns normally* → success → `STATUS_DONE`. + - `TaskRetry` (and any other/unexpected `Exception`, the safe default) → + retryable failure → `STATUS_SUSPENDED`, `retry += 1`, capped backoff. + - `TaskDefer(reason)` → not a failure, just "can't proceed yet" (node not + online, peer restart in flight, sibling task on same node) → + `STATUS_SUSPENDED`, **retry NOT consumed**, short re-poll. Replaces today's + scattered "suspend but don't increment retry" branches. + - `TaskAbort(reason)` → permanent / non-retryable (missing param, object not + found, "not needed") → `STATUS_DONE` with a failure/short-circuit result. + Replaces the handlers that currently mark DONE mid-body. +- Execution: `concurrency == 1` runs inline (serial); `> 1` uses a + `ThreadPoolExecutor` with a per-task inflight set and an optional + `exclusion_key(task)` per-key inflight set + capped exponential backoff — + generalizing `node_add`/`restart`. +- A `RunnerSpec` describing: `function_names`, `handler` (a void callable), + `is_eligible` (default `lambda task, cluster: True`), `interval` (each runner + keeps its current cadence), `concurrency`, `exclusion_key`. + +**B1…Bn — Migrate one runner per commit**, simplest → hardest, each preserving +behavior and keeping the module-level handler importable for the existing tests: + +1. `fdb_backup`, `jc_comp`, `replication_final`, `sync_lvol_del` (trivial serial) +2. `backup`, `cluster_expand` +3. `node_add` (concurrency opt-in, no exclusion key) +4. migration trio — `migration`, `new_dev_migration`, `failed_migration` + (serial; `get_active_node_mig_task` / same-node-sibling gating becomes the + spec's `is_eligible` predicate, not a concurrency `exclusion_key`) +5. `port_allow` (large; its `is_eligible` omits the `IN_ACTIVATION` check — the + documented opt-out — and keeps the recovery logic) +6. `restart` (concurrency + per-node exclusion + `fd_dead_recovery_allowed` / + `is_auto_restart_paused` gating; the driver's `exclusion_key` + backoff + subsumes `_node_inflight`/`_restart_next_attempt`. `is_auto_restart_paused` + becomes part of `is_eligible`) +7. `lvol_migration` (largest; migrate the loop/lease/retry shell only, leave the + domain snapshot-copy state machine intact. Its pre-run guard chain — node + status, `cluster.status in (ACTIVE, DEGRADED)`, open cluster-expand task — + becomes the spec's `is_eligible` predicate; unbounded `max_retry=-1` is kept) +8. `node_removal` (new upstream runner, `FN_NODE_REMOVAL`; already leased, + module-level loop, unbounded — a straightforward serial migration) +9. `batch_migration` (new upstream runner, `FN_LVOL_BATCH_MIG`, ~884 lines; + migrate the loop/lease shell, leave its group orchestration intact) + +**Upstream reconciliation (done during the 2026-07 rebase onto origin/main).** +Upstream independently reworked the runners: renamed `task_runner`→`process_task` +and added `tasks_controller.task_lease_heartbeat` (a lease-keepalive thread) to +the six long-blocking runners. This is now **folded into the B0 driver** — the +driver wraps every handler call in `task_lease_heartbeat`, so each migrated runner +gets keepalive for free and the per-runner boilerplate is deleted on migration. +The branch's original "Align task runner structure" commit was dropped as +superseded; A1/A2/A3 were re-applied on top of upstream (and A1/A3 extended to the +two new runners). Note: `tests/unit/tasks/test_retry_ceiling.py` **hangs on +origin/main itself** — upstream restructured the runners it drives without updating +it. It is stale and will be superseded by the driver's own tests as runners migrate +(each `main()` becomes `serve(SPEC)`); it should be skipped/removed rather than +propped up. + +Each `main()` collapses to `serve(SPEC)`. Each `task_runner(task)` becomes the +spec's **void handler**: it keeps only the domain work, returns `None` on +success, and raises `TaskRetry` / `TaskDefer` / `TaskAbort` (per B0) instead of +setting `task.status`/`retry` or calling `write_to_db`. Existing per-runner +mappings to preserve when translating: `jc_comp`'s "compression not needed" → +`TaskAbort`; "node not online" / "task on same node" → `TaskDefer`; RPC failure +→ `TaskRetry`. `restart`'s defer-vs-fail distinction (currently inferred by +comparing `retry` before/after) becomes explicit `TaskDefer` vs `TaskRetry`, +removing that bookkeeping. The module-level handler stays importable so +`tests/unit/tasks/` keeps calling it directly (tests assert on raised exceptions ++ driver-applied state rather than in-handler writes). + +--- + +## Files + +- **New:** `simplyblock_core/services/task_runner_base.py` (+ `tests/unit/tasks/test_task_runner_base.py`) +- **Renamed:** `tasks_runner_backup_merge.py` → `backup_merge_service.py` +- **Modified (all in `simplyblock_core/services/`):** `tasks_runner_fdb_backup`, + `_jc_comp`, `_replication_final`, `_sync_lvol_del`, `_backup`, `_cluster_expand`, + `_node_add`, `_migration`, `_new_dev_migration`, `_failed_migration`, + `_port_allow`, `_restart`, `_lvol_migration` +- **Modified:** `simplyblock_core/scripts/docker-compose-swarm.yml` (backup_merge command; no other command changes — filenames of converted runners stay the same) +- **Untouched:** `tasks_cluster_status.py` +- **Reused (no change):** `simplyblock_core/controllers/tasks_controller.py`, + `constants.py`, `storage_node_ops.py`, `controllers/snapshot_controller.py` + +## Verification + +Per the `tox-verify` skill — after each commit, targeted; full suite before finishing: + +1. `tox run-parallel -e lint,types` — must be green. +2. Targeted unit tests while iterating: + `tox run -e unit -- tests/unit/tasks/ tests/unit/test_task_lease.py \ + tests/unit/test_port_allow_recovery_refactor.py \ + tests/unit/test_drain_replaced_with_fixed_sleep.py \ + tests/unit/test_lvol_sync_op_task.py \ + tests/unit/tasks/test_task_runner_base.py` + The existing retry/lease tests must pass unchanged (they pin the standardized + semantics); extend them where new behavior warrants. New `test_task_runner_base.py` + asserts the exception vocabulary: a void return → DONE, `TaskRetry`/unexpected + `Exception` → SUSPENDED+retry+backoff, `TaskDefer` → SUSPENDED without + consuming a retry, `TaskAbort` → DONE; that an `is_eligible → False` task is + deferred without entering its handler or consuming a retry (default-`True` + spec always runs); and that a handler exception never escapes the loop while a + DB error does exit. +3. Import-smoke every converted runner (`python -c "import simplyblock_core.services.tasks_runner_X"`) + to confirm module-level definitions still load under the stubbed-fdb unit env. +4. `tox run -e unit` full unit tier; then the integration test that patches the + lvol-migration runner (`tests/integration/test_dual_fault_tolerance.py`) if + Docker + `libfdb_c` are available. +5. **DB-failure behavior (manual, infra-dependent):** run one converted runner + against a dev-compose FDB, stop FDB, and confirm the process exits non-zero + (immediate `sys.exit(1)`) rather than hanging — i.e. it surfaces as a Swarm + restart. Confirm a task-handler exception instead only suspends that task and + the loop keeps polling. + +### Rebase-friendliness note + +Commits are single-purpose and mostly additive (new module) or line-local +(loop-body edits), so conflicts against ongoing upstream runner fixes stay small +and localized — mirroring the resolution already done for +`tasks_runner_sync_lvol_del.py` on this branch. From ee0e34a208bd2ca9333437af81632dbeeded908f Mon Sep 17 00:00:00 2001 From: Max Schettler Date: Wed, 5 Aug 2026 13:12:14 +0200 Subject: [PATCH 07/25] Reset task result before each handler attempt Without this, a task that fails once and succeeds later finishes DONE while still carrying the previous attempt's failure message, because _succeed only fills in "completed" when the result is empty. --- simplyblock_core/services/task_runner_base.py | 5 +++++ tests/unit/tasks/test_task_runner_base.py | 22 +++++++++++++++++++ 2 files changed, 27 insertions(+) diff --git a/simplyblock_core/services/task_runner_base.py b/simplyblock_core/services/task_runner_base.py index 9c923523e9..2022027fd9 100644 --- a/simplyblock_core/services/task_runner_base.py +++ b/simplyblock_core/services/task_runner_base.py @@ -202,6 +202,11 @@ def _process(self, task: JobSchedule, cluster: Any) -> None: task.status = JobSchedule.STATUS_RUNNING task.write_to_db(db.kv_store) + # Drop the previous attempt's result so a task that fails and later + # succeeds does not finish carrying the stale failure message. Handlers + # that set a success message overwrite this; the rest get "completed". + task.function_result = "" + try: # Heartbeat the lease for the duration of the handler: TASK_LEASE_TTL # is far shorter than a node-add / restart / migration, so a lease diff --git a/tests/unit/tasks/test_task_runner_base.py b/tests/unit/tasks/test_task_runner_base.py index ffe556cae8..fd7498c5b4 100644 --- a/tests/unit/tasks/test_task_runner_base.py +++ b/tests/unit/tasks/test_task_runner_base.py @@ -118,6 +118,28 @@ def handler(_task): assert task.retry == 1 +def test_success_message_comes_from_the_handler(monkeypatch): + task = _task() + _wire(monkeypatch, task) + + def handler(t): + t.function_result = "Backup created" + + _runner(handler)._process(task, MagicMock()) + assert task.status == JobSchedule.STATUS_DONE + assert task.function_result == "Backup created" + + +def test_previous_failure_result_does_not_survive_a_later_success(monkeypatch): + task = _task(status=JobSchedule.STATUS_SUSPENDED, retry=1) + task.function_result = "rpc failed" + _wire(monkeypatch, task) + + _runner(MagicMock(return_value=None))._process(task, MagicMock()) + assert task.status == JobSchedule.STATUS_DONE + assert task.function_result == "completed" + + def test_abort_marks_done_with_reason(monkeypatch): task = _task() _wire(monkeypatch, task) From ef28b0561058e9cc7b72c16eb4dd2feaac7def9c Mon Sep 17 00:00:00 2001 From: Max Schettler Date: Wed, 5 Aug 2026 13:13:04 +0200 Subject: [PATCH 08/25] Migrate FDB backup runner onto the task runner The hand-written loop, lease claim, cancel/max-retry handling and status transitions all move to the driver; what remains is the backup call itself. A failed create_backup previously left the task RUNNING and re-ran it every cycle without consuming a retry, so its max_retry ceiling never applied. It is now a TaskRetry, which suspends the task, consumes a retry and backs off like every other runner's failure path. --- .../services/tasks_runner_fdb_backup.py | 74 ++++++------------- tests/unit/tasks/test_runner_specs.py | 61 +++++++++++++++ 2 files changed, 84 insertions(+), 51 deletions(-) create mode 100644 tests/unit/tasks/test_runner_specs.py diff --git a/simplyblock_core/services/tasks_runner_fdb_backup.py b/simplyblock_core/services/tasks_runner_fdb_backup.py index 91bbb4c5a5..c49c16afd0 100644 --- a/simplyblock_core/services/tasks_runner_fdb_backup.py +++ b/simplyblock_core/services/tasks_runner_fdb_backup.py @@ -1,11 +1,9 @@ # coding=utf-8 -import time - - -from simplyblock_core import db_controller, utils, constants -from simplyblock_core.controllers import fdb_backup_controller, tasks_controller +from simplyblock_core import db_controller, utils +from simplyblock_core.controllers import fdb_backup_controller from simplyblock_core.models.job_schedule import JobSchedule from simplyblock_core.models.cluster import Cluster +from simplyblock_core.services.task_runner_base import RunnerSpec, TaskRetry, serve logger = utils.get_logger(__name__) @@ -14,49 +12,23 @@ def process_fdb_backup_task(task): - task = db.get_task_by_id(task.uuid) - if task.canceled: - task.function_result = "canceled" - task.status = JobSchedule.STATUS_DONE - task.write_to_db(db.kv_store) - return - - if 0 <= task.max_retry <= task.retry: - task.function_result = "max retry reached, stopping task" - task.status = JobSchedule.STATUS_DONE - task.write_to_db(db.kv_store) - return - - if task.status != JobSchedule.STATUS_RUNNING: - task.status = JobSchedule.STATUS_RUNNING - task.write_to_db(db.kv_store) - - ret = fdb_backup_controller.create_backup(task.cluster_id) - if ret: - task.function_result = "Backup created" - task.status = JobSchedule.STATUS_DONE - task.write_to_db(db.kv_store) - - - -logger.info("Starting Tasks runner fdb backup...") - -while True: - clusters = db.get_clusters() - if not clusters: - logger.error("No clusters found!") - else: - for cl in clusters: - if cl.status == Cluster.STATUS_IN_ACTIVATION: - continue - - tasks = db.get_job_tasks(cl.get_id()) - for task in tasks: - if task.status != JobSchedule.STATUS_DONE: - if task.function_name == JobSchedule.FN_FDB_BACKUP: - if not tasks_controller.claim_task(task): - logger.info(f"FDB-backup task {task.uuid} owned by another runner host; skipping") - continue - process_fdb_backup_task(task) - - time.sleep(constants.TASK_EXEC_INTERVAL_SEC) + if not fdb_backup_controller.create_backup(task.cluster_id): + raise TaskRetry("failed to create backup") + + task.function_result = "Backup created" + + +SPEC = RunnerSpec( + name="tasks-runner-fdb-backup", + function_names=[JobSchedule.FN_FDB_BACKUP], + handler=process_fdb_backup_task, + is_eligible=lambda task, cluster: cluster.status != Cluster.STATUS_IN_ACTIVATION, +) + + +def main(): + serve(SPEC) + + +if __name__ == "__main__": + main() diff --git a/tests/unit/tasks/test_runner_specs.py b/tests/unit/tasks/test_runner_specs.py new file mode 100644 index 0000000000..ea09b9ad67 --- /dev/null +++ b/tests/unit/tasks/test_runner_specs.py @@ -0,0 +1,61 @@ +# coding=utf-8 +"""Per-runner tests for the runners migrated onto the shared driver. + +Each migrated runner is reduced to a :class:`RunnerSpec`: a void handler that +signals through ``TaskDefer`` / ``TaskRetry`` / ``TaskAbort``, plus an +eligibility predicate. These tests pin that translation — the loop, lease and +retry mechanics themselves are covered once in ``test_task_runner_base.py``. +""" +from unittest.mock import MagicMock + +import pytest + +from simplyblock_core.models.cluster import Cluster +from simplyblock_core.models.job_schedule import JobSchedule +import simplyblock_core.services.task_runner_base as trb + + +def _task(**params): + task = JobSchedule() + task.uuid = "task-1" + task.cluster_id = "cl-1" + task.node_id = "node-1" + task.function_params = params + return task + + +def _cluster(status=Cluster.STATUS_ACTIVE): + cluster = MagicMock() + cluster.status = status + return cluster + + +# -- fdb_backup ------------------------------------------------------------- + +@pytest.fixture +def fdb_backup(monkeypatch): + import simplyblock_core.services.tasks_runner_fdb_backup as runner + monkeypatch.setattr(runner, "fdb_backup_controller", MagicMock()) + return runner + + +def test_fdb_backup_success_reports_result(fdb_backup): + fdb_backup.fdb_backup_controller.create_backup.return_value = True + task = _task() + + assert fdb_backup.SPEC.handler(task) is None + fdb_backup.fdb_backup_controller.create_backup.assert_called_once_with("cl-1") + assert task.function_result == "Backup created" + + +def test_fdb_backup_failure_is_retryable(fdb_backup): + fdb_backup.fdb_backup_controller.create_backup.return_value = False + + with pytest.raises(trb.TaskRetry): + fdb_backup.SPEC.handler(_task()) + + +def test_fdb_backup_skips_clusters_in_activation(fdb_backup): + task = _task() + assert fdb_backup.SPEC.is_eligible(task, _cluster()) is True + assert fdb_backup.SPEC.is_eligible(task, _cluster(Cluster.STATUS_IN_ACTIVATION)) is False From e12506ea4deea0f6d14bddd691265a4755838550 Mon Sep 17 00:00:00 2001 From: Max Schettler Date: Wed, 5 Aug 2026 13:20:39 +0200 Subject: [PATCH 09/25] Migrate JC compression resume runner onto the task runner The loop, lease, cancel/max-retry handling and every task write move to the driver; the handler keeps only the resume decision and signals through TaskDefer (node offline, sibling task on the node, cluster not fully online), TaskAbort (node gone, compression not needed) and TaskRetry (resume failed). This also fixes the not-all-nodes-online branch, whose `continue` resumed the enclosing node scan rather than skipping the task, so a single offline node suspended the task and then issued the RPC anyway. Runners on the driver no longer carry a retry counter, so they drop out of the retry-ceiling harness's source-discovered parametrization. They are now listed explicitly instead, guarded by a test that they really did hand the counter over. --- .../services/tasks_runner_jc_comp.py | 167 ++++++------------ tests/unit/tasks/test_retry_ceiling.py | 40 +++-- tests/unit/tasks/test_runner_specs.py | 109 ++++++++++++ 3 files changed, 188 insertions(+), 128 deletions(-) diff --git a/simplyblock_core/services/tasks_runner_jc_comp.py b/simplyblock_core/services/tasks_runner_jc_comp.py index 22099afa9e..af28c2e638 100644 --- a/simplyblock_core/services/tasks_runner_jc_comp.py +++ b/simplyblock_core/services/tasks_runner_jc_comp.py @@ -1,13 +1,17 @@ # coding=utf-8 -import time - - from simplyblock_core import db_controller, utils from simplyblock_core.controllers import tasks_controller from simplyblock_core.models.job_schedule import JobSchedule from simplyblock_core.models.cluster import Cluster from simplyblock_core.models.storage_node import StorageNode from simplyblock_core.release_upgrades import jc_compression_upgrade +from simplyblock_core.services.task_runner_base import ( + RunnerSpec, + TaskAbort, + TaskDefer, + TaskRetry, + serve, +) logger = utils.get_logger(__name__) @@ -15,120 +19,51 @@ db = db_controller.DBController() +def process_task(task): + # Release-upgrade guard (remove with the jc_compression_upgrade plugin): + # resumes are held until `cluster upgrade-complete`. + if jc_compression_upgrade.resume_is_held(db.get_cluster_by_id(task.cluster_id)): + raise TaskDefer("JC compression resume held: cluster upgrade in progress") + + try: + node = db.get_storage_node_by_id(task.node_id) + except KeyError: + raise TaskAbort("node not found") + + if node.status != StorageNode.STATUS_ONLINE: + raise TaskDefer(f"Node is {node.status}, retry task") + + if tasks_controller.get_active_node_tasks(task.cluster_id, task.node_id): + raise TaskDefer("Task found on same node") + + if any(n.status != StorageNode.STATUS_ONLINE + for n in db.get_storage_nodes_by_cluster_id(node.cluster_id)): + raise TaskDefer("Not all nodes are online, can not resume JC compression") + + logger.info("no task found on same node, resuming compression") + jm_vuid = task.function_params.get("jm_vuid", node.jm_vuid) + ret, err = node.rpc_client(timeout=5, retry=2).jc_suspend_compression( + jm_vuid=jm_vuid, suspend=False) + + if not ret: + if err: + raise TaskAbort(f"JC {node.jm_vuid} compression not needed") + raise TaskRetry("JC comp resume failed, retry task") + + task.function_result = f"JC {node.jm_vuid} compression resumed on node" + + +SPEC = RunnerSpec( + name="tasks-runner-jc-comp", + function_names=[JobSchedule.FN_JC_COMP_RESUME], + handler=process_task, + is_eligible=lambda task, cluster: cluster.status != Cluster.STATUS_IN_ACTIVATION, + interval=60, +) + + def main(): - logger.info("Starting Tasks runner...") - while True: - clusters = db.get_clusters() - if not clusters: - logger.error("No clusters found!") - else: - for cl in clusters: - if cl.status == Cluster.STATUS_IN_ACTIVATION: - continue - - tasks = db.get_job_tasks(cl.get_id(), reverse=False) - for task in tasks: - - if task.function_name == JobSchedule.FN_JC_COMP_RESUME: - if task.status != JobSchedule.STATUS_DONE: - if not tasks_controller.claim_task(task): - logger.info(f"JC-comp task {task.uuid} owned by another runner host; skipping") - continue - - # get new task object because it could be changed from cancel task - task = db.get_task_by_id(task.uuid) - - if task.canceled: - task.function_result = "canceled" - task.status = JobSchedule.STATUS_DONE - task.write_to_db(db.kv_store) - continue - - # Release-upgrade guard (remove with the - # jc_compression_upgrade plugin): resumes are - # held until `cluster upgrade-complete`. - if jc_compression_upgrade.resume_is_held(cl): - msg = "JC compression resume held: cluster upgrade in progress" - logger.info(msg) - # Only write when something actually changes. The - # hold lasts until `cluster upgrade-complete` runs, - # so an unconditional write here costs one DB write - # per held task per poll for the whole upgrade. - if (task.status != JobSchedule.STATUS_SUSPENDED - or task.function_result != msg): - task.function_result = msg - task.status = JobSchedule.STATUS_SUSPENDED - task.write_to_db(db.kv_store) - continue - - if 0 <= task.max_retry <= task.retry: - task.function_result = "max retry reached, stopping task" - task.status = JobSchedule.STATUS_DONE - task.write_to_db(db.kv_store) - continue - - try: - node = db.get_storage_node_by_id(task.node_id) - except KeyError: - task.function_result = "node not found" - task.status = JobSchedule.STATUS_DONE - task.write_to_db(db.kv_store) - continue - - if node.status != StorageNode.STATUS_ONLINE: - msg = f"Node is {node.status}, retry task" - logger.info(msg) - task.function_result = msg - task.status = JobSchedule.STATUS_SUSPENDED - task.write_to_db(db.kv_store) - continue - - node_task = tasks_controller.get_active_node_tasks(task.cluster_id, task.node_id) - if node_task: - msg="Task found on same node" - logger.info(msg) - task.retry += 1 - task.function_result = msg - task.status = JobSchedule.STATUS_SUSPENDED - task.write_to_db(db.kv_store) - else: - logger.info("no task found on same node, resuming compression") - node = db.get_storage_node_by_id(task.node_id) - for n in db.get_storage_nodes_by_cluster_id(node.cluster_id): - if n.status != StorageNode.STATUS_ONLINE: - msg = "Not all nodes are online, can not resume JC compression" - logger.info(msg) - task.function_result = msg - task.status = JobSchedule.STATUS_SUSPENDED - task.write_to_db(db.kv_store) - continue - - rpc_client = node.rpc_client(timeout=5, retry=2) - jm_vuid = node.jm_vuid - if "jm_vuid" in task.function_params: - jm_vuid = task.function_params["jm_vuid"] - try: - ret, err = rpc_client.jc_suspend_compression(jm_vuid=jm_vuid, suspend=False) - except Exception as e: - logger.error(e) - continue - if ret: - task.function_result = f"JC {node.jm_vuid} compression resumed on node" - task.status = JobSchedule.STATUS_DONE - task.write_to_db(db.kv_store) - elif err: - task.function_result = f"JC {node.jm_vuid} compression not needed" - task.status = JobSchedule.STATUS_DONE - task.write_to_db(db.kv_store) - else: - msg = "JC comp resume failed, retry task" - logger.info(msg) - task.retry += 1 - task.function_result = msg - task.status = JobSchedule.STATUS_SUSPENDED - task.write_to_db(db.kv_store) - - time.sleep(60) + serve(SPEC) if __name__ == "__main__": diff --git a/tests/unit/tasks/test_retry_ceiling.py b/tests/unit/tasks/test_retry_ceiling.py index 5744b0c23b..6423c7fca8 100644 --- a/tests/unit/tasks/test_retry_ceiling.py +++ b/tests/unit/tasks/test_retry_ceiling.py @@ -334,16 +334,6 @@ def _spec_replication_final(runner, monkeypatch): return task -def _spec_jc_comp(runner, monkeypatch): - task = _make_task(JobSchedule.FN_JC_COMP_RESUME) - _wire_base(runner, monkeypatch, task) - # A task is always active on the same node -> resume is deferred, retry each - # poll (this is the branch that increments task.retry). - monkeypatch.setattr(runner.tasks_controller, "get_active_node_tasks", - lambda *a, **k: [MagicMock()]) - return task - - def _spec_restart(runner, monkeypatch): task = _make_task(JobSchedule.FN_NODE_RESTART) _db, _cluster, node = _wire_base(runner, monkeypatch, task) @@ -422,7 +412,6 @@ def _get_node(node_id): "tasks_runner_cluster_expand.py": _spec_cluster_expand, "tasks_runner_node_add.py": _spec_node_add, "tasks_runner_replication_final.py": _spec_replication_final, - "tasks_runner_jc_comp.py": _spec_jc_comp, "tasks_runner_restart.py": _spec_restart, "tasks_runner_batch_migration.py": _spec_batch_migration, } @@ -435,6 +424,18 @@ def _get_node(node_id): "tasks_runner_backup.py": "driven via process_task in test_backup_* above", } +# Runners migrated onto the shared driver (``task_runner_base``). They no longer +# own a loop or a retry counter — the driver enforces the ceiling for all of +# them at once, covered by tests/unit/tasks/test_task_runner_base.py. They are +# therefore not discovered by _retry_driven_runner_files(); listing them here +# keeps that disappearance deliberate rather than silent, and +# test_migrated_runners_delegate_retry below pins that they really did hand the +# retry counter over. +_DRIVER_MIGRATED = { + "tasks_runner_fdb_backup.py", + "tasks_runner_jc_comp.py", +} + # Runners that increment task.retry but are intentionally UNBOUNDED: the # migration family is created with max_retry=-1 and gates retries on resource # recovery (see _migration_retry_allowed) rather than a fixed count. Value is @@ -506,6 +507,21 @@ def test_registries_are_not_stale(): names = {p.name for p in _runner_files()} listed = (set(_MAIN_DRIVEN_SPECS) | set(_COVERED_ELSEWHERE) - | set(INTENTIONALLY_UNBOUNDED)) + | set(INTENTIONALLY_UNBOUNDED) + | _DRIVER_MIGRATED) missing = listed - names assert not missing, f"listed runners no longer exist: {missing}" + + +@pytest.mark.parametrize("name", sorted(_DRIVER_MIGRATED)) +def test_migrated_runners_delegate_retry(name): + """A runner listed as migrated must not have kept a retry counter of its + own: the driver owns task.retry, and a runner that still increments it + would be applying two ceilings at once.""" + source = (Path(_runner_files()[0]).parent / name).read_text(encoding='utf-8') + assert not _INCREMENTS_RETRY.search(source), ( + f"{name} is listed as migrated to task_runner_base but still increments " + "task.retry itself") + assert "serve(SPEC)" in source, ( + f"{name} is listed as migrated to task_runner_base but does not run the " + "shared driver") diff --git a/tests/unit/tasks/test_runner_specs.py b/tests/unit/tasks/test_runner_specs.py index ea09b9ad67..84bbacf29d 100644 --- a/tests/unit/tasks/test_runner_specs.py +++ b/tests/unit/tasks/test_runner_specs.py @@ -11,7 +11,9 @@ import pytest from simplyblock_core.models.cluster import Cluster +from simplyblock_core.models.storage_node import StorageNode from simplyblock_core.models.job_schedule import JobSchedule +from simplyblock_core.release_upgrades import jc_compression_upgrade import simplyblock_core.services.task_runner_base as trb @@ -59,3 +61,110 @@ def test_fdb_backup_skips_clusters_in_activation(fdb_backup): task = _task() assert fdb_backup.SPEC.is_eligible(task, _cluster()) is True assert fdb_backup.SPEC.is_eligible(task, _cluster(Cluster.STATUS_IN_ACTIVATION)) is False + + +# -- jc_comp ---------------------------------------------------------------- + +def _node(status=StorageNode.STATUS_ONLINE, jm_vuid=7): + node = MagicMock() + node.status = status + node.jm_vuid = jm_vuid + node.cluster_id = "cl-1" + return node + + +@pytest.fixture +def jc_comp(monkeypatch): + import simplyblock_core.services.tasks_runner_jc_comp as runner + db = MagicMock() + monkeypatch.setattr(runner, "db", db) + monkeypatch.setattr(runner, "tasks_controller", MagicMock()) + runner.tasks_controller.get_active_node_tasks.return_value = [] + db.get_storage_nodes_by_cluster_id.return_value = [_node()] + # An unstubbed MagicMock attribute auto-creates, so `release_upgrade_state` + # would answer .get() with a truthy mock and hold every resume. + db.get_cluster_by_id.return_value.release_upgrade_state = {} + return runner + + +def test_jc_comp_resumes_compression(jc_comp): + node = _node() + jc_comp.db.get_storage_node_by_id.return_value = node + node.rpc_client.return_value.jc_suspend_compression.return_value = (True, None) + task = _task() + + assert jc_comp.SPEC.handler(task) is None + node.rpc_client.return_value.jc_suspend_compression.assert_called_once_with( + jm_vuid=7, suspend=False) + assert task.function_result == "JC 7 compression resumed on node" + + +def test_jc_comp_prefers_the_task_jm_vuid(jc_comp): + node = _node() + jc_comp.db.get_storage_node_by_id.return_value = node + node.rpc_client.return_value.jc_suspend_compression.return_value = (True, None) + + jc_comp.SPEC.handler(_task(jm_vuid=42)) + node.rpc_client.return_value.jc_suspend_compression.assert_called_once_with( + jm_vuid=42, suspend=False) + + +def test_jc_comp_aborts_on_missing_node(jc_comp): + jc_comp.db.get_storage_node_by_id.side_effect = KeyError("node-1") + with pytest.raises(trb.TaskAbort): + jc_comp.SPEC.handler(_task()) + + +def test_jc_comp_defers_while_the_release_upgrade_holds_resumes(jc_comp): + node = _node() + jc_comp.db.get_storage_node_by_id.return_value = node + jc_comp.db.get_cluster_by_id.return_value.release_upgrade_state = { + jc_compression_upgrade.STATE_KEY: True, + } + + with pytest.raises(trb.TaskDefer): + jc_comp.SPEC.handler(_task()) + node.rpc_client.assert_not_called() + + +def test_jc_comp_defers_while_node_is_offline(jc_comp): + jc_comp.db.get_storage_node_by_id.return_value = _node(StorageNode.STATUS_OFFLINE) + with pytest.raises(trb.TaskDefer): + jc_comp.SPEC.handler(_task()) + + +def test_jc_comp_defers_while_another_task_runs_on_the_node(jc_comp): + jc_comp.db.get_storage_node_by_id.return_value = _node() + jc_comp.tasks_controller.get_active_node_tasks.return_value = [_task()] + with pytest.raises(trb.TaskDefer): + jc_comp.SPEC.handler(_task()) + + +def test_jc_comp_defers_unless_every_cluster_node_is_online(jc_comp): + node = _node() + jc_comp.db.get_storage_node_by_id.return_value = node + jc_comp.db.get_storage_nodes_by_cluster_id.return_value = [ + _node(StorageNode.STATUS_OFFLINE), node, + ] + + with pytest.raises(trb.TaskDefer): + jc_comp.SPEC.handler(_task()) + node.rpc_client.assert_not_called() + + +def test_jc_comp_aborts_when_compression_is_not_needed(jc_comp): + node = _node() + jc_comp.db.get_storage_node_by_id.return_value = node + node.rpc_client.return_value.jc_suspend_compression.return_value = (False, "not needed") + + with pytest.raises(trb.TaskAbort): + jc_comp.SPEC.handler(_task()) + + +def test_jc_comp_retries_when_resume_fails(jc_comp): + node = _node() + jc_comp.db.get_storage_node_by_id.return_value = node + node.rpc_client.return_value.jc_suspend_compression.return_value = (False, None) + + with pytest.raises(trb.TaskRetry): + jc_comp.SPEC.handler(_task()) From 161b25c435566c5ff45e606349d5feacbba4e4f9 Mon Sep 17 00:00:00 2001 From: Max Schettler Date: Wed, 5 Aug 2026 13:24:12 +0200 Subject: [PATCH 10/25] Migrate replication cutover runner onto the task runner MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The loop, lease, cancel/max-retry handling and the _finalize state machine move to the driver. Every branch _finalize was called with failure was a suspend-and-retry, so they become TaskRetry; the success path keeps only the replication-state update and the timestamps. The failure path's fixed 3s sleep is dropped in favour of the driver's per-task backoff, which no longer stalls the other tasks in the cycle. Drops the stale comment claiming a missing source node proceeds with a best-effort ANA flip — the code has always retried instead. --- .../tasks_runner_replication_final.py | 280 ++++++++---------- ...lication_relationship_and_delete_source.py | 28 +- .../test_tasks_runner_replication_final.py | 78 ++--- tests/unit/tasks/test_retry_ceiling.py | 13 +- 4 files changed, 177 insertions(+), 222 deletions(-) diff --git a/simplyblock_core/services/tasks_runner_replication_final.py b/simplyblock_core/services/tasks_runner_replication_final.py index 66df139c7f..fee5725907 100644 --- a/simplyblock_core/services/tasks_runner_replication_final.py +++ b/simplyblock_core/services/tasks_runner_replication_final.py @@ -24,13 +24,13 @@ import uuid as uuid_lib from datetime import datetime -from simplyblock_core import constants, db_controller, utils -from simplyblock_core.controllers import tasks_controller +from simplyblock_core import db_controller, utils from simplyblock_core.models.job_schedule import JobSchedule from simplyblock_core.models.lvol_model import LVol, LVolReplication from simplyblock_core.models.snapshot import SnapShot from simplyblock_core.models.storage_node import StorageNode from simplyblock_core.services import replication_final_step +from simplyblock_core.services.task_runner_base import RunnerSpec, TaskDefer, TaskRetry, serve logger = utils.get_logger(__name__) utils.init_sentry_sdk(__name__) @@ -38,145 +38,121 @@ db = db_controller.DBController() -def _finalize(task, ok, err): - if ok: - replication_id = task.function_params.get("replication_id") - final_state = task.function_params.get("final_state", LVolReplication.STATE_CUTOVER_DONE) - if replication_id: - try: - rep = db.get_lvol_replication_by_id(replication_id) - rep.state = final_state - rep.write_to_db(db.kv_store) - except Exception as e: - logger.error(f"Failed to update replication state: {e}") - task.function_result = "cutover done" - task.status = JobSchedule.STATUS_DONE - task.function_params["end_time"] = int(time.time()) - task.write_to_db(db.kv_store) - - # The hand-off is complete: the target serves the data from here on, - # so the SOURCE must stop replicating. Nothing else clears its cadence - # config, and a retired source otherwise keeps taking internal - # snapshots and shipping them to the very target it handed off to - # (observed 2026-08-21: replication_final done "cutover done" while - # the source volumes kept replicating). In-flight transfers drain - # naturally; this stops NEW cadence snapshots at the gate the monitor - # reads (do_replicate / replication_interval_min). - try: - src_lvol = db.get_lvol_by_id(task.function_params.get("lvol_id")) - if src_lvol.do_replicate: - src_lvol.do_replicate = False - src_lvol.replication_interval_min = 0 - src_lvol.replication_policy_id = "" - src_lvol.write_to_db() - logger.info(f"Cutover done: stopped replication on source " - f"volume {src_lvol.get_id()}") - except KeyError: - pass # source already gone (e.g. deleted out of band) - except Exception as e: - logger.error(f"Could not stop replication on the source after " - f"cutover: {e}") - - # Optional migration semantics: the source volume has served its - # purpose once the client runs on the target, so `replication-commit - # --delete-source` retires it here — strictly AFTER the cutover state - # is durable, so a crash in between leaves a completed cutover with - # the source still present (retryable by hand), never a deleted - # source with an uncommitted cutover. The relationship record is what - # later look-ups (target-by-source, active side) resolve through, and - # it survives the volume's deletion. - if task.function_params.get("delete_source"): - src_lvol_id = task.function_params.get("lvol_id") - try: - from simplyblock_core.controllers import lvol_controller - src_lvol = db.get_lvol_by_id(src_lvol_id) - logger.info(f"Cutover committed with --delete-source: deleting " - f"source volume {src_lvol_id}") - lvol_controller.delete_lvol(src_lvol) - except Exception as e: - # The cutover itself succeeded; a failed source delete is - # reported loudly but does not un-succeed the task. - logger.error(f"Source volume {src_lvol_id} could not be " - f"deleted after the cutover: {e}") - return True - - task.function_result = err or "cutover failed, retrying" - task.status = JobSchedule.STATUS_SUSPENDED - task.retry += 1 - task.write_to_db(db.kv_store) - return False +def _record_cutover_done(task): + replication_id = task.function_params.get("replication_id") + if not replication_id: + return + + final_state = task.function_params.get("final_state", LVolReplication.STATE_CUTOVER_DONE) + try: + rep = db.get_lvol_replication_by_id(replication_id) + rep.state = final_state + rep.write_to_db(db.kv_store) + except Exception as e: + logger.error(f"Failed to update replication state: {e}") + + +def _stop_source_replication(task): + """The hand-off is complete: the target serves the data from here on, so the + SOURCE must stop replicating. Nothing else clears its cadence config, and a + retired source otherwise keeps taking internal snapshots and shipping them + to the very target it handed off to (observed 2026-08-21: replication_final + done "cutover done" while the source volumes kept replicating). In-flight + transfers drain naturally; this stops NEW cadence snapshots at the gate the + monitor reads (do_replicate / replication_interval_min). + """ + try: + src_lvol = db.get_lvol_by_id(task.function_params.get("lvol_id")) + if src_lvol.do_replicate: + src_lvol.do_replicate = False + src_lvol.replication_interval_min = 0 + src_lvol.replication_policy_id = "" + src_lvol.write_to_db() + logger.info(f"Cutover done: stopped replication on source " + f"volume {src_lvol.get_id()}") + except KeyError: + pass # source already gone (e.g. deleted out of band) + except Exception as e: + logger.error(f"Could not stop replication on the source after " + f"cutover: {e}") + + +def _delete_source_if_requested(task): + """Optional migration semantics: the source volume has served its purpose + once the client runs on the target, so `replication-commit --delete-source` + retires it here — strictly AFTER the cutover state is durable, so a crash in + between leaves a completed cutover with the source still present (retryable + by hand), never a deleted source with an uncommitted cutover. The + relationship record is what later look-ups (target-by-source, active side) + resolve through, and it survives the volume's deletion. + """ + if not task.function_params.get("delete_source"): + return + + src_lvol_id = task.function_params.get("lvol_id") + try: + from simplyblock_core.controllers import lvol_controller + src_lvol = db.get_lvol_by_id(src_lvol_id) + logger.info(f"Cutover committed with --delete-source: deleting " + f"source volume {src_lvol_id}") + lvol_controller.delete_lvol(src_lvol) + except Exception as e: + # The cutover itself succeeded; a failed source delete is reported + # loudly but does not un-succeed the task. + logger.error(f"Source volume {src_lvol_id} could not be " + f"deleted after the cutover: {e}") def task_runner(task: JobSchedule): params = task.function_params lvol_id = params.get("lvol_id") if not lvol_id: - return _finalize(task, False, "missing lvol_id in task params") - - if (0 <= task.max_retry <= task.retry) or task.canceled is True: - task.function_result = "task cancelled" if task.canceled else "max retry reached" - task.status = JobSchedule.STATUS_DONE - task.write_to_db(db.kv_store) - return True + raise TaskRetry("missing lvol_id in task params") try: lvol = db.get_lvol_by_id(lvol_id) tgt_node = db.get_storage_node_by_id(params["tgt_node_id"]) except KeyError as e: - return _finalize(task, False, f"object not found: {e}") + raise TaskRetry(f"object not found: {e}") + + if tgt_node.status != StorageNode.STATUS_ONLINE: + raise TaskRetry("target node not online, retrying") - # The source may be gone entirely (fail-over after cluster loss); cutover - # proceeds with a best-effort ANA flip in that case. - src_node = None try: src_node = db.get_storage_node_by_id(params["src_node_id"]) except KeyError: - pass + raise TaskRetry("source node not found for cutover") - if tgt_node.status != StorageNode.STATUS_ONLINE: - task.function_result = "target node not online, retrying" - task.status = JobSchedule.STATUS_SUSPENDED - task.retry += 1 - task.write_to_db(db.kv_store) - return False - - if src_node is None: - return _finalize(task, False, "source node not found for cutover") - - if task.status in [JobSchedule.STATUS_NEW, JobSchedule.STATUS_SUSPENDED, JobSchedule.STATUS_RUNNING]: - task.status = JobSchedule.STATUS_RUNNING - task.function_params.setdefault("start_time", int(time.time())) - task.write_to_db(db.kv_store) - - # ---- SHRINK PHASE ----------------------------------------------- # - if "shrink_snap_id" in params and params.get("shrink_round", 0) > 0: - done, err = _shrink_step(task, lvol) - if err: - return _finalize(task, False, err) - if not done: - # waiting on replication of the current shrink snapshot; come - # back next pass WITHOUT burning a retry (bounded by deadline) - task.status = JobSchedule.STATUS_SUSPENDED - task.write_to_db(db.kv_store) - return False - - # ---- CUTOVER PHASE (immediately after the last shrink round) ---- # - if "tgt_lvol_composite" not in params: - err = _prepare_cutover(task, lvol, src_node, tgt_node) - if err: - return _finalize(task, False, err) - params = task.function_params - - try: - ok, err = replication_final_step.run_cutover( - src_node, tgt_node, lvol, - params["tgt_lvol_composite"], params["tgt_map_id"], - params["tgt_snap_composite"], operation=params.get("operation", "replicate")) - except Exception as e: - logger.error(f"Cutover raised: {e}", exc_info=True) - return _finalize(task, False, str(e)) - return _finalize(task, ok, err) - return True + params.setdefault("start_time", int(time.time())) + + # ---- SHRINK PHASE --------------------------------------------------- # + if "shrink_snap_id" in params and params.get("shrink_round", 0) > 0: + _shrink_step(task, lvol) + + # ---- CUTOVER PHASE (immediately after the last shrink round) -------- # + if "tgt_lvol_composite" not in params: + _prepare_cutover(task, lvol, src_node, tgt_node) + + ok, err = replication_final_step.run_cutover( + src_node, tgt_node, lvol, + params["tgt_lvol_composite"], params["tgt_map_id"], + params["tgt_snap_composite"], operation=params.get("operation", "replicate")) + if not ok: + raise TaskRetry(err or "cutover failed, retrying") + + _record_cutover_done(task) + params["end_time"] = int(time.time()) + task.function_result = "cutover done" + + _stop_source_replication(task) + _delete_source_if_requested(task) + + +SPEC = RunnerSpec( + name="tasks-runner-replication-final", + function_names=[JobSchedule.FN_REPLICATION_FINAL], + handler=task_runner, +) SHRINK_ROUNDS = 2 @@ -185,28 +161,29 @@ def task_runner(task: JobSchedule): def _shrink_step(task, lvol): """Advance the delta-shrink state machine one step. - Returns (done, error): done=True when all rounds are replicated and the - cutover may start IMMEDIATELY; error aborts the task. + Returns once all rounds are replicated and the cutover may start + IMMEDIATELY. Raises :class:`TaskDefer` while a round is still replicating — + waiting for the target is not a failure, so it must not consume a retry; the + wait is bounded by ``shrink_deadline`` instead. """ params = task.function_params if int(time.time()) > params.get("shrink_deadline", 0): - return False, "shrink phase timed out waiting for replication" + raise TaskRetry("shrink phase timed out waiting for replication") snap_id = params["shrink_snap_id"] try: snap = db.get_snapshot_by_id(snap_id) except KeyError: - return False, f"shrink snapshot {snap_id} disappeared" + raise TaskRetry(f"shrink snapshot {snap_id} disappeared") # target_replicated_snap_uuid is set at replicate-finish AFTER the target # copy is chained and converted — replicated AND converted in one signal. if not snap.target_replicated_snap_uuid: - task.function_result = (f"shrink round {params['shrink_round']}: waiting " - f"for {snap_id[:8]} to replicate") - return False, None + raise TaskDefer(f"shrink round {params['shrink_round']}: waiting " + f"for {snap_id[:8]} to replicate") if params["shrink_round"] >= SHRINK_ROUNDS: - return True, None + return # Round replicated — IMMEDIATELY take the next snapshot: its delta covers # only the wait window of the previous round. @@ -215,12 +192,10 @@ def _shrink_step(task, lvol): lvol.get_id(), f"repl_commit_{uuid_lib.uuid4()}", snap_type=SnapShot.TYPE_INTERNAL) if err: - return False, f"shrink round {params['shrink_round'] + 1} snapshot failed: {err}" + raise TaskRetry(f"shrink round {params['shrink_round'] + 1} snapshot failed: {err}") params["shrink_round"] += 1 params["shrink_snap_id"] = new_snap - task.function_result = f"shrink round {params['shrink_round']}: snapshot taken" - task.write_to_db(db.kv_store) - return False, None + raise TaskDefer(f"shrink round {params['shrink_round']}: snapshot taken") def _prepare_cutover(task, lvol, src_node, tgt_node): @@ -240,7 +215,7 @@ def _prepare_cutover(task, lvol, src_node, tgt_node): new_lvol, snapshot, error = lvol_controller._clone_from_last_replicated( db, lvol.get_id(), lvol, tgt_node, target_pool_uuid, src_node.cluster_id) if error: - return f"cutover clone failed: {error}" + raise TaskRetry(f"cutover clone failed: {error}") new_lvol.status = LVol.STATUS_ONLINE new_lvol.write_to_db(db.kv_store) @@ -251,7 +226,7 @@ def _prepare_cutover(task, lvol, src_node, tgt_node): if tgt_map_id is None: lvol_controller.delete_lvol_from_node(new_lvol, tgt_node) db.release_lvol_ns_slot(new_lvol) - return "could not resolve target map_id" + raise TaskRetry("could not resolve target map_id") rep = LVolReplication() rep.uuid = str(uuid_lib.uuid4()) @@ -273,32 +248,15 @@ def _prepare_cutover(task, lvol, src_node, tgt_node): "tgt_snap_composite": snapshot.snap_bdev, "replication_id": rep.get_id(), }) + # Checkpoint the task itself, against the driver's usual ownership of task + # writes: the clone and the LVolReplication record above already exist, so a + # crash before the cutover completes must not let the next attempt build a + # second clone. task.write_to_db(db.kv_store) - return None def main(): - logger.info("Starting replication-final tasks runner...") - while True: - clusters = db.get_clusters() - for cl in clusters: - for task in db.get_job_tasks(cl.get_id(), reverse=False): - if task.function_name != JobSchedule.FN_REPLICATION_FINAL: - continue - if task.status == JobSchedule.STATUS_DONE: - continue - task = db.get_task_by_id(task.uuid) - if not tasks_controller.claim_task(task): - logger.info(f"Replication-final task {task.uuid} owned by another runner host; skipping") - continue - try: - res = task_runner(task) - except Exception as e: - logger.error(f"replication-final task {task.uuid} failed: {e}", exc_info=True) - res = False - if not res: - time.sleep(3) - time.sleep(constants.TASK_EXEC_INTERVAL_SEC) + serve(SPEC) if __name__ == "__main__": diff --git a/simplyblock_core/test/test_replication_relationship_and_delete_source.py b/simplyblock_core/test/test_replication_relationship_and_delete_source.py index a297391a0e..f91ee5ae9b 100644 --- a/simplyblock_core/test/test_replication_relationship_and_delete_source.py +++ b/simplyblock_core/test/test_replication_relationship_and_delete_source.py @@ -211,36 +211,38 @@ def _delete(lvol, **kw): monkeypatch.setattr(lvol_controller, "delete_lvol", _delete) task = _Task(delete_source) - ok = trf._finalize(task, True, "") - return ok, task, events + # The handler's success path. The task's terminal state is written by the + # task runner afterwards, so what this observes is the ordering of the + # source retirement against the durable cutover state. + trf._record_cutover_done(task) + task.function_result = "cutover done" + trf._stop_source_replication(task) + trf._delete_source_if_requested(task) + return task, events def test_cutover_stops_replication_on_the_source(monkeypatch): """Observed 2026-08-21: after "cutover done" the source kept taking and replicating cadence snapshots. The hand-off must stop the source.""" - ok, _task, events = _run_finalize(monkeypatch, delete_source=False) - assert ok is True + _task, events = _run_finalize(monkeypatch, delete_source=False) assert ("src_config", False, 0, "") in events, "the source's replication config must be cleared at cutover" def test_source_deleted_only_after_cutover_state_is_durable(monkeypatch): - ok, task, events = _run_finalize(monkeypatch, delete_source=True) - assert ok is True + task, events = _run_finalize(monkeypatch, delete_source=True) assert ("delete", "SRC1") in events state_at = events.index(("state", LVolReplication.STATE_CUTOVER_DONE)) delete_at = events.index(("delete", "SRC1")) assert state_at < delete_at, "the cutover state must be durable BEFORE the delete" - assert task.status == JobSchedule.STATUS_DONE + assert task.function_result == "cutover done" def test_source_kept_without_the_flag(monkeypatch): - ok, _task, events = _run_finalize(monkeypatch, delete_source=False) - assert ok is True + _task, events = _run_finalize(monkeypatch, delete_source=False) assert not any(e[0] == "delete" for e in events) def test_failed_source_delete_does_not_unsucceed_the_cutover(monkeypatch): - ok, task, _events = _run_finalize(monkeypatch, delete_source=True, - delete_raises=True) - assert ok is True - assert task.status == JobSchedule.STATUS_DONE + task, _events = _run_finalize(monkeypatch, delete_source=True, + delete_raises=True) + assert task.function_result == "cutover done" diff --git a/simplyblock_core/test/test_tasks_runner_replication_final.py b/simplyblock_core/test/test_tasks_runner_replication_final.py index 787ec9f69e..5b812309e7 100644 --- a/simplyblock_core/test/test_tasks_runner_replication_final.py +++ b/simplyblock_core/test/test_tasks_runner_replication_final.py @@ -1,7 +1,16 @@ -"""D6 unit tests for the replication-final task runner lifecycle.""" +"""D6 unit tests for the replication-final task handler. + +The runner sits on the shared driver (``task_runner_base``), so the handler +is void: it returns on success, raises ``TaskRetry`` for every outcome the +driver should suspend and re-attempt, and ``TaskDefer`` while the delta-shrink +phase waits on the target without burning a retry. Task status/retry transitions +themselves belong to the driver and are covered by +tests/unit/tasks/test_task_runner_base.py. +""" import pytest from simplyblock_core.services import tasks_runner_replication_final as runner +from simplyblock_core.services.task_runner_base import TaskDefer, TaskRetry from simplyblock_core.models.job_schedule import JobSchedule from simplyblock_core.models.lvol_model import LVol, LVolReplication from simplyblock_core.models.storage_node import StorageNode @@ -76,58 +85,57 @@ def _run_cutover(src, tgt, lvol, comp, map_id, snap, operation="replicate"): return calls -def test_happy_path_marks_done_and_updates_state(monkeypatch): +def test_happy_path_returns_and_updates_state(monkeypatch): rep = LVolReplication() rep.state = LVolReplication.STATE_CUTOVER_PENDING nodes = {"S1": _node("S1"), "T1": _node("T1")} calls = _install(monkeypatch, nodes, rep, (True, None)) - res = runner.task_runner(_task()) + task = _task() + assert runner.task_runner(task) is None - assert res is True assert len(calls) == 1 assert calls[0][5] == "replicate" assert rep.state == LVolReplication.STATE_CUTOVER_DONE + assert task.function_result == "cutover done" + assert task.function_params["start_time"] <= task.function_params["end_time"] -def test_failure_suspends_and_retries(monkeypatch): +def test_failed_cutover_is_retryable(monkeypatch): rep = LVolReplication() nodes = {"S1": _node("S1"), "T1": _node("T1")} _install(monkeypatch, nodes, rep, (False, "boom")) - task = _task() - res = runner.task_runner(task) - - assert res is False - assert task.status == JobSchedule.STATUS_SUSPENDED - assert task.retry == 1 - assert task.function_result == "boom" + with pytest.raises(TaskRetry, match="boom"): + runner.task_runner(_task()) -def test_max_retry_marks_done_without_cutover(monkeypatch): +def test_target_offline_is_retryable_without_cutover(monkeypatch): rep = LVolReplication() - nodes = {"S1": _node("S1"), "T1": _node("T1")} + nodes = {"S1": _node("S1"), "T1": _node("T1", status=StorageNode.STATUS_OFFLINE)} calls = _install(monkeypatch, nodes, rep, (True, None)) - task = _task() - task.retry = 5 # == max_retry - res = runner.task_runner(task) + with pytest.raises(TaskRetry): + runner.task_runner(_task()) + assert calls == [] - assert res is True - assert task.status == JobSchedule.STATUS_DONE - assert calls == [] # cutover never attempted +def test_missing_source_node_is_retryable_without_cutover(monkeypatch): + rep = LVolReplication() + calls = _install(monkeypatch, {"T1": _node("T1")}, rep, (True, None)) -def test_target_offline_suspends(monkeypatch): + with pytest.raises(TaskRetry, match="source node not found"): + runner.task_runner(_task()) + assert calls == [] + + +def test_missing_lvol_id_is_retryable(monkeypatch): rep = LVolReplication() - nodes = {"S1": _node("S1"), "T1": _node("T1", status=StorageNode.STATUS_OFFLINE)} + nodes = {"S1": _node("S1"), "T1": _node("T1")} calls = _install(monkeypatch, nodes, rep, (True, None)) - task = _task() - res = runner.task_runner(task) - - assert res is False - assert task.status == JobSchedule.STATUS_SUSPENDED + with pytest.raises(TaskRetry, match="missing lvol_id"): + runner.task_runner(_task(lvol_id="")) assert calls == [] @@ -181,9 +189,8 @@ def test_shrink_waits_until_replicated(monkeypatch): runner, task = _mk(monkeypatch, {"S1": _ShrinkSnap(replicated=False)}, {"shrink_round": 1, "shrink_snap_id": "S1", "shrink_deadline": 2**60}) - done, err = runner._shrink_step(task, _ShrinkLvol()) - assert (done, err) == (False, None) - assert "waiting" in task.function_result + with pytest.raises(TaskDefer, match="waiting"): + runner._shrink_step(task, _ShrinkLvol()) def test_shrink_takes_next_snapshot_immediately(monkeypatch): @@ -198,8 +205,8 @@ def _add(lid, name, snap_type="user"): import simplyblock_core.controllers.snapshot_controller as sc monkeypatch.setattr(sc, "add", _add) - done, err = runner._shrink_step(task, _ShrinkLvol()) - assert (done, err) == (False, None) + with pytest.raises(TaskDefer): + runner._shrink_step(task, _ShrinkLvol()) assert taken == [("LV1", "internal")] or taken[0][0] == "LV1" assert task.function_params["shrink_round"] == 2 assert task.function_params["shrink_snap_id"] == "S2" @@ -209,8 +216,7 @@ def test_shrink_completes_after_last_round(monkeypatch): runner, task = _mk(monkeypatch, {"S2": _ShrinkSnap(replicated=True)}, {"shrink_round": runner_rounds(), "shrink_snap_id": "S2", "shrink_deadline": 2**60}) - done, err = runner._shrink_step(task, _ShrinkLvol()) - assert (done, err) == (True, None), "cutover must start IMMEDIATELY after the last round" + runner._shrink_step(task, _ShrinkLvol()) # returns => cutover starts IMMEDIATELY def runner_rounds(): @@ -222,5 +228,5 @@ def test_shrink_deadline_aborts(monkeypatch): runner, task = _mk(monkeypatch, {"S1": _ShrinkSnap(replicated=False)}, {"shrink_round": 1, "shrink_snap_id": "S1", "shrink_deadline": 1}) - done, err = runner._shrink_step(task, _ShrinkLvol()) - assert done is False and err and "timed out" in err + with pytest.raises(TaskRetry, match="timed out"): + runner._shrink_step(task, _ShrinkLvol()) diff --git a/tests/unit/tasks/test_retry_ceiling.py b/tests/unit/tasks/test_retry_ceiling.py index 6423c7fca8..c9847c1dbb 100644 --- a/tests/unit/tasks/test_retry_ceiling.py +++ b/tests/unit/tasks/test_retry_ceiling.py @@ -323,17 +323,6 @@ def _spec_node_add(runner, monkeypatch): return task -def _spec_replication_final(runner, monkeypatch): - task = _make_task( - JobSchedule.FN_REPLICATION_FINAL, - lvol_id="lv-1", tgt_node_id="tgt-1", src_node_id="src-1") - db, _cluster, node = _wire_base(runner, monkeypatch, task) - # Target node never comes online -> cutover cannot proceed, retry each poll. - node.status = StorageNode.STATUS_OFFLINE - db.get_lvol_by_id.return_value = MagicMock() - return task - - def _spec_restart(runner, monkeypatch): task = _make_task(JobSchedule.FN_NODE_RESTART) _db, _cluster, node = _wire_base(runner, monkeypatch, task) @@ -411,7 +400,6 @@ def _get_node(node_id): _MAIN_DRIVEN_SPECS = { "tasks_runner_cluster_expand.py": _spec_cluster_expand, "tasks_runner_node_add.py": _spec_node_add, - "tasks_runner_replication_final.py": _spec_replication_final, "tasks_runner_restart.py": _spec_restart, "tasks_runner_batch_migration.py": _spec_batch_migration, } @@ -434,6 +422,7 @@ def _get_node(node_id): _DRIVER_MIGRATED = { "tasks_runner_fdb_backup.py", "tasks_runner_jc_comp.py", + "tasks_runner_replication_final.py", } # Runners that increment task.retry but are intentionally UNBOUNDED: the From cf7898f108a5dbf41a9c31d95eb180f110349508 Mon Sep 17 00:00:00 2001 From: Max Schettler Date: Wed, 5 Aug 2026 13:29:57 +0200 Subject: [PATCH 11/25] Migrate lvol sync runner onto the task runner MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both task families it serves move to the driver's contract. The sync-op handler comes back from tasks_controller, where it only lived because the runner used to execute its loop at import time; with the loop behind serve(SPEC) it belongs next to the runner it serves. The sync-delete failure paths (RPC exception, delete rejected) suspended the task without ever incrementing retry, so its declared max_retry=10 could never bind and a permanently failing delete re-issued forever. They are TaskRetry now, which makes the ceiling effective. The primary's del-sync lock is released through the driver's new on_finish hook, so it is also freed on the terminal paths the handler never sees — cancellation and the retry ceiling — instead of only on the two the old inline code happened to cover. --- .../controllers/tasks_controller.py | 84 ------ simplyblock_core/services/task_runner_base.py | 21 +- .../services/tasks_runner_sync_lvol_del.py | 266 ++++++++++-------- tests/unit/tasks/test_retry_ceiling.py | 1 + tests/unit/tasks/test_task_runner_base.py | 47 ++++ tests/unit/test_lifecycle_alert_events.py | 7 +- tests/unit/test_lvol_sync_op_task.py | 75 +++-- 7 files changed, 271 insertions(+), 230 deletions(-) diff --git a/simplyblock_core/controllers/tasks_controller.py b/simplyblock_core/controllers/tasks_controller.py index a69cad6610..455b6c0c39 100644 --- a/simplyblock_core/controllers/tasks_controller.py +++ b/simplyblock_core/controllers/tasks_controller.py @@ -982,90 +982,6 @@ def add_lvol_sync_op_task(cluster_id, node_id, lvol_id, op, secondary_index=0): max_retry=-1) -def run_lvol_sync_op_task(task): - """Execute one FN_LVOL_SYNC_OP task (called by the sync runner's loop; - lives here so it is importable without triggering the runner's - module-level loop). Idempotent; never raises.""" - from simplyblock_core import storage_node_ops - from simplyblock_core.models.lvol_model import LVol - - def _finish(result): - task.function_result = result - task.status = JobSchedule.STATUS_DONE - task.write_to_db(db.kv_store) - - def _defer(result): - task.function_result = result - task.status = JobSchedule.STATUS_SUSPENDED - task.write_to_db(db.kv_store) - - try: - if task.canceled: - _finish("canceled") - return - - lvol_id = task.function_params.get("lvol_id") - op = task.function_params.get("op") - try: - lvol = db.get_lvol_by_id(lvol_id) - except KeyError: - _finish("lvol no longer exists") - return - if lvol.status == LVol.STATUS_IN_DELETION: - _finish("lvol is being deleted") - return - if lvol.status != LVol.STATUS_ONLINE: - _defer(f"lvol status is {lvol.status}, retrying") - return - - try: - node = db.get_storage_node_by_id(task.node_id) - except KeyError: - _finish("node no longer exists") - return - if node.get_id() not in lvol.nodes: - _finish("node no longer hosts this lvol (topology moved)") - return - if node.status != StorageNode.STATUS_ONLINE: - _defer(f"node is {node.status}, retrying") - return - if storage_node_ops.get_restart_phase(task.node_id, lvol.lvs_name): - # The owning flow (restart/activation/expansion) re-registers - # lvols itself; re-check once it has released the LVS. - _defer("LVS owned by a restart/activation/expansion, retrying") - return - - if task.status != JobSchedule.STATUS_RUNNING: - task.status = JobSchedule.STATUS_RUNNING - task.write_to_db(db.kv_store) - - if op == "register": - ok, err = storage_node_ops.repair_lvol_registration_on_non_leader( - lvol, node, task.function_params.get("secondary_index", 0)) - if ok: - _finish(f"registered lvol {lvol_id} on {task.node_id}") - else: - _defer(f"registration failed: {err}") - elif op == "resize": - # Converge to the CURRENT DB size — resize_lvol persists the new - # size after the fan-out, so this always applies the latest - # target even if the lvol was resized again meanwhile. - size_in_mib = utils.convert_size(lvol.size, 'MiB') - ret = node.rpc_client(timeout=10, retry=2).bdev_lvol_resize( - f"{lvol.lvs_name}/{lvol.lvol_bdev}", size_in_mib) - if ret: - _finish(f"resized lvol {lvol_id} on {task.node_id} to {size_in_mib} MiB") - else: - _defer("resize RPC failed, retrying") - else: - _finish(f"unknown op {op!r}") - except Exception as e: - logger.error(f"lvol sync-op task {task.uuid} failed: {e}") - try: - _defer(f"error: {e}") - except Exception as defer_e: - logger.debug(f"lvol sync-op task {task.uuid}: _defer fallback also failed: {defer_e}") - def get_lvol_sync_del_task(cluster_id, node_id, lvol_bdev_name=None): tasks = db.get_job_tasks(cluster_id) for task in tasks: diff --git a/simplyblock_core/services/task_runner_base.py b/simplyblock_core/services/task_runner_base.py index 2022027fd9..21cbd929c4 100644 --- a/simplyblock_core/services/task_runner_base.py +++ b/simplyblock_core/services/task_runner_base.py @@ -88,6 +88,11 @@ class RunnerSpec: # exclusion_key() is equal never run at the same time (e.g. one restart per # node). Ignored when concurrency == 1. exclusion_key: Optional[Callable[[JobSchedule], Any]] = None + # Optional cleanup, called once the task has reached STATUS_DONE and been + # written — whichever way it got there (handler success, TaskAbort, cancel + # or retry ceiling). For releasing state the task held, which would + # otherwise leak on the terminal paths the handler never sees. + on_finish: Optional[Callable[[JobSchedule], None]] = None def __post_init__(self) -> None: if self.concurrency < 1: @@ -233,16 +238,28 @@ def _succeed(self, task: JobSchedule) -> None: task.status = JobSchedule.STATUS_DONE if not task.function_result: task.function_result = "completed" - task.write_to_db(db.kv_store) - self._forget(task.uuid) + self._write_terminal(task) def _finish(self, task: JobSchedule, result: str) -> None: """Terminal DONE for a non-handler-success reason (canceled, max retry, abort).""" task.function_result = result task.status = JobSchedule.STATUS_DONE + self._write_terminal(task) + + def _write_terminal(self, task: JobSchedule) -> None: task.write_to_db(db.kv_store) self._forget(task.uuid) + if self.spec.on_finish is None: + return + # Cleanup runs after the terminal write, so a hook that inspects the + # task's own state (a lock held until no active task remains) sees it + # as finished. A failing hook must not take the loop down with it. + try: + self.spec.on_finish(task) + except Exception as e: # noqa: BLE001 - cleanup failure is not fatal + logger.error(f"{self.spec.name}: task {task.uuid} on_finish failed: {e}") + logger.exception(e) def _defer(self, task: JobSchedule, reason: str) -> None: if reason: diff --git a/simplyblock_core/services/tasks_runner_sync_lvol_del.py b/simplyblock_core/services/tasks_runner_sync_lvol_del.py index 1068134c74..8eb13d3871 100644 --- a/simplyblock_core/services/tasks_runner_sync_lvol_del.py +++ b/simplyblock_core/services/tasks_runner_sync_lvol_del.py @@ -1,18 +1,37 @@ # coding=utf-8 -import time +"""Task runner for the deferred per-node lvol operations. + +Two task families share this runner, both of them work that could not be done +inline on the owning node at the time it was requested: + +- ``FN_LVOL_SYNC_OP`` — re-apply a create-registration or a resize on a + non-leader node (incident 2026-07-10: an in-memory deferral queue was never + drained, so a volume's tertiary subsystem was never created). +- ``FN_LVOL_SYNC_DEL`` — delete a replica bdev on a secondary node, holding the + primary's del-sync lock until the task ends. +""" from typing import Optional -from simplyblock_core import db_controller, utils -from simplyblock_core.controllers import events_controller, snapshot_controller, tasks_controller +from simplyblock_core import db_controller, storage_node_ops, utils +from simplyblock_core.controllers import events_controller, snapshot_controller from simplyblock_core.models.job_schedule import JobSchedule from simplyblock_core.models.cluster import Cluster +from simplyblock_core.models.lvol_model import LVol from simplyblock_core.models.storage_node import StorageNode +from simplyblock_core.services.task_runner_base import ( + RunnerSpec, + TaskAbort, + TaskDefer, + TaskRetry, + serve, +) logger = utils.get_logger(__name__) # get DB controller db = db_controller.DBController() + def get_primary_node(task) -> Optional[StorageNode]: if "primary_node" in task.function_params: return db.get_storage_node_by_id(task.function_params["primary_node"]) @@ -23,18 +42,71 @@ def get_primary_node(task) -> Optional[StorageNode]: return None +def _run_sync_op(task): + lvol_id = task.function_params.get("lvol_id") + op = task.function_params.get("op") + + try: + lvol = db.get_lvol_by_id(lvol_id) + except KeyError: + raise TaskAbort("lvol no longer exists") + + if lvol.status == LVol.STATUS_IN_DELETION: + raise TaskAbort("lvol is being deleted") + if lvol.status != LVol.STATUS_ONLINE: + raise TaskDefer(f"lvol status is {lvol.status}, retrying") + + try: + node = db.get_storage_node_by_id(task.node_id) + except KeyError: + raise TaskAbort("node no longer exists") + + if node.get_id() not in lvol.nodes: + raise TaskAbort("node no longer hosts this lvol (topology moved)") + if node.status != StorageNode.STATUS_ONLINE: + raise TaskDefer(f"node is {node.status}, retrying") + if storage_node_ops.get_restart_phase(task.node_id, lvol.lvs_name): + # The owning flow (restart/activation/expansion) re-registers + # lvols itself; re-check once it has released the LVS. + raise TaskDefer("LVS owned by a restart/activation/expansion, retrying") + + if op == "register": + ok, err = storage_node_ops.repair_lvol_registration_on_non_leader( + lvol, node, task.function_params.get("secondary_index", 0)) + if not ok: + raise TaskDefer(f"registration failed: {err}") + task.function_result = f"registered lvol {lvol_id} on {task.node_id}" + elif op == "resize": + # Converge to the CURRENT DB size — resize_lvol persists the new + # size after the fan-out, so this always applies the latest + # target even if the lvol was resized again meanwhile. + size_in_mib = utils.convert_size(lvol.size, 'MiB') + if not node.rpc_client(timeout=10, retry=2).bdev_lvol_resize( + f"{lvol.lvs_name}/{lvol.lvol_bdev}", size_in_mib): + raise TaskDefer("resize RPC failed, retrying") + task.function_result = f"resized lvol {lvol_id} on {task.node_id} to {size_in_mib} MiB" + else: + raise TaskAbort(f"unknown op {op!r}") + + +# Last failure message reported per delete task, so a node that stays down does +# not write one identical event per 3s poll. The driver clears ``function_result`` +# before every attempt, so the previous message cannot be read off the task. +_last_sync_del_failure: dict = {} + + def _log_sync_delete_failure(task, node, lvol_bdev_name, msg): """Record a failed sync delete in the cluster event log. This is the case an operator cannot see from the volume list alone: the async delete already SUCCEEDED, so the data is going away, but a node still holds its replica bdev and the volume is pinned in_deletion until this - task drains. Emitted only when the failure message CHANGES (first failure, - or a different error), not on every 3s retry -- a node that stays down - would otherwise write thousands of identical events. + task drains. """ - if task.function_result == msg: + if _last_sync_del_failure.get(task.uuid) == msg: return + _last_sync_del_failure[task.uuid] = msg + try: events_controller.log_event_cluster( cluster_id=node.cluster_id, @@ -51,121 +123,73 @@ def _log_sync_delete_failure(task, node, lvol_bdev_name, msg): logger.warning(f"Could not log sync-delete failure event: {event_error}") +def _run_sync_del(task): + try: + node = db.get_storage_node_by_id(task.node_id) + except KeyError: + raise TaskAbort("node not found") + + if node.status not in [StorageNode.STATUS_DOWN, StorageNode.STATUS_ONLINE]: + raise TaskDefer(f"Node is {node.status}, retry task") + + lvol_bdev_name = task.function_params["lvol_bdev_name"] + logger.info(f"Sync delete bdev: {lvol_bdev_name} from node: {node.get_id()}") + try: + # Per-node lvstore lock: the sync delete mutates the replica blob tree + # and must not interleave with a create/register of another object on + # this node. + with snapshot_controller.lvstore_op_lock( + node.cluster_id, + lvol_bdev_name.split("/")[0], + node_id=node.get_id()): + ret, err = node.rpc_client().delete_lvol(lvol_bdev_name, sync=True) + except Exception as e: + msg = f"Sync delete of {lvol_bdev_name} on {node.get_id()} failed: {e}; will retry" + _log_sync_delete_failure(task, node, lvol_bdev_name, msg) + raise TaskRetry(msg) + + if not ret: + if "code" not in err or err["code"] != -19: + msg = f"Failed to sync delete bdev: {lvol_bdev_name} from node: {node.get_id()}" + _log_sync_delete_failure(task, node, lvol_bdev_name, msg) + raise TaskRetry(msg) + logger.error(f"Sync delete completed with error: {err}") + + task.function_result = f"bdev {lvol_bdev_name} deleted" + + +def process_task(task): + if task.function_name == JobSchedule.FN_LVOL_SYNC_OP: + _run_sync_op(task) + else: + _run_sync_del(task) + + +def release_del_sync_lock(task): + """Free the primary's del-sync lock once the delete task is over, however + it ended — the lock is keyed on there being no active task left.""" + if task.function_name != JobSchedule.FN_LVOL_SYNC_DEL: + return + + _last_sync_del_failure.pop(task.uuid, None) + + primary_node = get_primary_node(task) + if primary_node: + primary_node.lvol_del_sync_lock_reset() + + +SPEC = RunnerSpec( + name="tasks-runner-sync-lvol-del", + function_names=[JobSchedule.FN_LVOL_SYNC_OP, JobSchedule.FN_LVOL_SYNC_DEL], + handler=process_task, + on_finish=release_del_sync_lock, + is_eligible=lambda task, cluster: cluster.status != Cluster.STATUS_IN_ACTIVATION, + interval=3, +) + + def main(): - logger.info("Starting Tasks runner...") - - while True: - clusters = db.get_clusters() - if not clusters: - logger.error("No clusters found!") - else: - for cl in clusters: - if cl.status == Cluster.STATUS_IN_ACTIVATION: - continue - - tasks = db.get_job_tasks(cl.get_id(), reverse=False) - for task in tasks: - if task.function_name == JobSchedule.FN_LVOL_SYNC_OP: - if task.status != JobSchedule.STATUS_DONE: - if not tasks_controller.claim_task(task): - logger.info(f"LVol sync task {task.uuid} owned by another runner host; skipping") - continue - # Re-read (it may have been canceled concurrently). - task = db.get_task_by_id(task.uuid) - if task.status == JobSchedule.STATUS_DONE: - continue - try: - tasks_controller.run_lvol_sync_op_task(task) - except Exception as e: - logger.error(f"lvol sync-op task {task.uuid} crashed: {e}") - continue - - if task.function_name == JobSchedule.FN_LVOL_SYNC_DEL: - if task.status != JobSchedule.STATUS_DONE: - if not tasks_controller.claim_task(task): - logger.info(f"LVol sync task {task.uuid} owned by another runner host; skipping") - continue - - # get new task object because it could be changed from cancel task - task = db.get_task_by_id(task.uuid) - - if task.canceled: - task.function_result = "canceled" - task.status = JobSchedule.STATUS_DONE - task.write_to_db(db.kv_store) - primary_node = get_primary_node(task) - if primary_node: - primary_node.lvol_del_sync_lock_reset() - continue - - node = db.get_storage_node_by_id(task.node_id) - - if not node: - task.function_result = "node not found" - task.status = JobSchedule.STATUS_DONE - task.write_to_db(db.kv_store) - primary_node = db.get_storage_node_by_id(task.function_params["primary_node"]) - primary_node.lvol_del_sync_lock_reset() - continue - - if node.status not in [StorageNode.STATUS_DOWN, StorageNode.STATUS_ONLINE]: - msg = f"Node is {node.status}, retry task" - logger.info(msg) - task.function_result = msg - task.status = JobSchedule.STATUS_SUSPENDED - task.write_to_db(db.kv_store) - continue - - if task.status != JobSchedule.STATUS_RUNNING: - task.status = JobSchedule.STATUS_RUNNING - task.write_to_db(db.kv_store) - - lvol_bdev_name = task.function_params["lvol_bdev_name"] - - logger.info(f"Sync delete bdev: {lvol_bdev_name} from node: {node.get_id()}") - try: - # Per-node lvstore lock: the sync delete mutates - # the replica blob tree and must not interleave - # with a create/register of another object on - # this node. The try also keeps a dead node from - # killing the runner: on 2026-07-16 an unhandled - # RPCException ('connection error') here took the - # whole service down and no deferred sync delete - # ever ran again. - with snapshot_controller.lvstore_op_lock( - node.cluster_id, - lvol_bdev_name.split("/")[0], - node_id=node.get_id()): - ret, err = node.rpc_client().delete_lvol(lvol_bdev_name, sync=True) - except Exception as e: - msg = (f"Sync delete of {lvol_bdev_name} on {node.get_id()} " - f"failed: {e}; will retry") - logger.error(msg) - _log_sync_delete_failure(task, node, lvol_bdev_name, msg) - task.function_result = msg - task.status = JobSchedule.STATUS_SUSPENDED - task.write_to_db(db.kv_store) - continue - if not ret: - if "code" in err and err["code"] == -19: - logger.error(f"Sync delete completed with error: {err}") - else: - msg = f"Failed to sync delete bdev: {lvol_bdev_name} from node: {node.get_id()}" - logger.error(msg) - _log_sync_delete_failure(task, node, lvol_bdev_name, msg) - task.function_result = msg - task.status = JobSchedule.STATUS_SUSPENDED - task.write_to_db(db.kv_store) - continue - - task.function_result = f"bdev {lvol_bdev_name} deleted" - task.status = JobSchedule.STATUS_DONE - task.write_to_db(db.kv_store) - primary_node = get_primary_node(task) - if primary_node: - primary_node.lvol_del_sync_lock_reset() - - time.sleep(3) + serve(SPEC) if __name__ == "__main__": diff --git a/tests/unit/tasks/test_retry_ceiling.py b/tests/unit/tasks/test_retry_ceiling.py index c9847c1dbb..d99c490acf 100644 --- a/tests/unit/tasks/test_retry_ceiling.py +++ b/tests/unit/tasks/test_retry_ceiling.py @@ -423,6 +423,7 @@ def _get_node(node_id): "tasks_runner_fdb_backup.py", "tasks_runner_jc_comp.py", "tasks_runner_replication_final.py", + "tasks_runner_sync_lvol_del.py", } # Runners that increment task.retry but are intentionally UNBOUNDED: the diff --git a/tests/unit/tasks/test_task_runner_base.py b/tests/unit/tasks/test_task_runner_base.py index fd7498c5b4..bc3a2509bc 100644 --- a/tests/unit/tasks/test_task_runner_base.py +++ b/tests/unit/tasks/test_task_runner_base.py @@ -152,6 +152,53 @@ def handler(_task): assert task.function_result == "missing param" +# -- terminal cleanup hook -------------------------------------------------- + +@pytest.mark.parametrize("handler,expected", [ + (MagicMock(return_value=None), "completed"), + (MagicMock(side_effect=trb.TaskAbort("gone")), "gone"), +]) +def test_on_finish_runs_for_every_terminal_outcome(monkeypatch, handler, expected): + task = _task() + _wire(monkeypatch, task) + on_finish = MagicMock() + + _runner(handler, on_finish=on_finish)._process(task, MagicMock()) + + on_finish.assert_called_once_with(task) + assert task.status == JobSchedule.STATUS_DONE + assert task.function_result == expected + + +def test_on_finish_runs_when_the_handler_is_never_reached(monkeypatch): + task = _task(canceled=True) + _wire(monkeypatch, task) + on_finish = MagicMock() + + _runner(MagicMock(), on_finish=on_finish)._process(task, MagicMock()) + on_finish.assert_called_once_with(task) + + +def test_on_finish_does_not_run_for_a_suspended_task(monkeypatch): + task = _task() + _wire(monkeypatch, task) + on_finish = MagicMock() + + _runner(MagicMock(side_effect=trb.TaskDefer("later")), + on_finish=on_finish)._process(task, MagicMock()) + on_finish.assert_not_called() + + +def test_failing_on_finish_does_not_break_the_task(monkeypatch): + task = _task() + _wire(monkeypatch, task) + + runner = _runner(MagicMock(return_value=None), + on_finish=MagicMock(side_effect=RuntimeError("cleanup boom"))) + runner._process(task, MagicMock()) # must not raise + assert task.status == JobSchedule.STATUS_DONE + + # -- pre-run skip-gates ----------------------------------------------------- def test_ineligible_skips_without_claim_or_write(monkeypatch): diff --git a/tests/unit/test_lifecycle_alert_events.py b/tests/unit/test_lifecycle_alert_events.py index af3d3b72a9..b965d07818 100644 --- a/tests/unit/test_lifecycle_alert_events.py +++ b/tests/unit/test_lifecycle_alert_events.py @@ -30,7 +30,12 @@ class TestSyncDeleteFailureEvent(unittest.TestCase): def _task(self, previous_result=""): task = MagicMock() - task.function_result = previous_result + task.uuid = "task-1" + # The task runner clears function_result before every attempt, so the + # "same failure again" check reads the runner's own last-message map. + sync_del._last_sync_del_failure.clear() + if previous_result: + sync_del._last_sync_del_failure[task.uuid] = previous_result return task def _node(self): diff --git a/tests/unit/test_lvol_sync_op_task.py b/tests/unit/test_lvol_sync_op_task.py index 2e2e57191f..821d2adee0 100644 --- a/tests/unit/test_lvol_sync_op_task.py +++ b/tests/unit/test_lvol_sync_op_task.py @@ -5,6 +5,11 @@ ``_restart_op_queues`` deferral (incident 2026-07-10: a create-registration queued in the webappapi's in-memory queue was never drained; the volume's tertiary subsystem was never created). + +The runner sits on the shared driver, so its handler is void: it returns on +success and raises ``TaskDefer`` (retry later) / ``TaskAbort`` (permanently +obsolete). Task status transitions belong to the driver and are covered by +tests/unit/tasks/test_task_runner_base.py. """ import unittest @@ -14,6 +19,8 @@ from simplyblock_core.models.job_schedule import JobSchedule from simplyblock_core.models.lvol_model import LVol from simplyblock_core.models.storage_node import StorageNode +from simplyblock_core.services import tasks_runner_sync_lvol_del as runner +from simplyblock_core.services.task_runner_base import TaskAbort, TaskDefer def _task(op="register", lvol_id="lv-1", node_id="node-2", canceled=False, @@ -50,7 +57,7 @@ def _node(status=StorageNode.STATUS_ONLINE, node_id="node-2"): class TestRunLvolSyncOpTask(unittest.TestCase): def setUp(self): - patcher = patch.object(tasks_controller, "db") + patcher = patch.object(runner, "db") self.db = patcher.start() self.addCleanup(patcher.stop) @@ -62,7 +69,7 @@ def _run(self, task, lvol=None, node=None, phase=""): patch("simplyblock_core.storage_node_ops." "repair_lvol_registration_on_non_leader", return_value=(True, None)) as repair: - tasks_controller.run_lvol_sync_op_task(task) + runner.SPEC.handler(task) return repair def test_register_repairs_and_completes(self): @@ -70,47 +77,71 @@ def test_register_repairs_and_completes(self): repair = self._run(task) repair.assert_called_once() self.assertEqual(repair.call_args[0][2], 1) # secondary_index - self.assertEqual(task.status, JobSchedule.STATUS_DONE) + self.assertIn("registered lvol", task.function_result) - def test_deleted_lvol_completes_without_action(self): + def test_deleted_lvol_aborts_without_action(self): task = _task() self.db.get_lvol_by_id.side_effect = KeyError() - tasks_controller.run_lvol_sync_op_task(task) - self.assertEqual(task.status, JobSchedule.STATUS_DONE) - self.assertIn("no longer exists", task.function_result) + with self.assertRaises(TaskAbort) as ctx: + runner.SPEC.handler(task) + self.assertIn("no longer exists", str(ctx.exception)) def test_offline_node_defers(self): - task = _task() - self._run(task, node=_node(StorageNode.STATUS_OFFLINE)) - self.assertEqual(task.status, JobSchedule.STATUS_SUSPENDED) + with self.assertRaises(TaskDefer): + self._run(_task(), node=_node(StorageNode.STATUS_OFFLINE)) def test_owned_lvs_defers(self): - task = _task() - self._run(task, phase=StorageNode.RESTART_PHASE_BLOCKED) - self.assertEqual(task.status, JobSchedule.STATUS_SUSPENDED) + with self.assertRaises(TaskDefer): + self._run(_task(), phase=StorageNode.RESTART_PHASE_BLOCKED) - def test_node_dropped_from_topology_completes(self): - task = _task(node_id="node-9") - self._run(task, node=_node(node_id="node-9")) - self.assertEqual(task.status, JobSchedule.STATUS_DONE) - self.assertIn("no longer hosts", task.function_result) + def test_node_dropped_from_topology_aborts(self): + with self.assertRaises(TaskAbort) as ctx: + self._run(_task(node_id="node-9"), node=_node(node_id="node-9")) + self.assertIn("no longer hosts", str(ctx.exception)) def test_resize_converges_to_current_db_size(self): task = _task(op="resize") - lvol = _lvol() node = _node() rpc = node.rpc_client.return_value rpc.bdev_lvol_resize.return_value = True - self.db.get_lvol_by_id.return_value = lvol + self.db.get_lvol_by_id.return_value = _lvol() self.db.get_storage_node_by_id.return_value = node with patch("simplyblock_core.storage_node_ops.get_restart_phase", return_value=""): - tasks_controller.run_lvol_sync_op_task(task) + runner.SPEC.handler(task) rpc.bdev_lvol_resize.assert_called_once() args = rpc.bdev_lvol_resize.call_args[0] self.assertEqual(args[0], "LVS_6/LVOL_1") self.assertEqual(args[1], 4096) # 4 GiB in MiB - self.assertEqual(task.status, JobSchedule.STATUS_DONE) + self.assertIn("resized lvol", task.function_result) + + def test_unknown_op_aborts(self): + with self.assertRaises(TaskAbort): + self._run(_task(op="nonsense")) + + +class TestDelSyncLockRelease(unittest.TestCase): + """The primary's del-sync lock is held for the lifetime of the delete task, + so it must be released on every terminal path — including the ones the + handler never reaches (cancel, retry ceiling).""" + + def test_lock_released_for_a_finished_delete_task(self): + task = MagicMock(spec=JobSchedule) + task.function_name = JobSchedule.FN_LVOL_SYNC_DEL + task.function_params = {"primary_node": "node-1"} + task.node_id = "node-2" + + with patch.object(runner, "db") as db: + primary = _node(node_id="node-1") + db.get_storage_node_by_id.return_value = primary + runner.SPEC.on_finish(task) + + primary.lvol_del_sync_lock_reset.assert_called_once() + + def test_sync_op_task_does_not_touch_the_lock(self): + with patch.object(runner, "db") as db: + runner.SPEC.on_finish(_task()) + db.get_storage_node_by_id.assert_not_called() class TestAddTaskDedup(unittest.TestCase): From fefd4fe78e1e657f357cbec603695b5dc08fd980 Mon Sep 17 00:00:00 2001 From: Max Schettler Date: Wed, 5 Aug 2026 13:37:33 +0200 Subject: [PATCH 12/25] Migrate backup and cluster expand runners onto the task runner Both keep their domain step machines; what goes is the surrounding lifecycle. The backup runner's three families (backup, restore, merge) become void handlers over a shared dispatch, with the node lookup and transfer-state poll factored out since all three did them identically. _terminate_task becomes the driver's on_finish hook. It was previously reached only from the timeout and the retry ceiling, so a cancelled backup task left its backup PENDING forever and a cancelled merge left the old backup stuck in MERGING; a restore that gave up on a missing node left its lvol RESTORING. All of those now finalize. Each branch was already guarded on the resource still being in flight, so a successful task passes through untouched. Cluster expand loses its inner per-task loop, which re-ran a single expansion back-to-back with its own backoff and starved every other task in the cycle. The driver polls it once per cycle and backs off the same way. --- .../services/tasks_runner_backup.py | 509 +++++++----------- .../services/tasks_runner_cluster_expand.py | 170 ++---- .../test_tasks_runner_cluster_expand.py | 54 +- tests/unit/tasks/test_retry_ceiling.py | 127 +---- tests/unit/tasks/test_runner_specs.py | 178 ++++++ 5 files changed, 443 insertions(+), 595 deletions(-) diff --git a/simplyblock_core/services/tasks_runner_backup.py b/simplyblock_core/services/tasks_runner_backup.py index 35d8240e75..a001e9af45 100644 --- a/simplyblock_core/services/tasks_runner_backup.py +++ b/simplyblock_core/services/tasks_runner_backup.py @@ -6,74 +6,79 @@ - FN_BACKUP: perform an S3 backup from a snapshot - FN_BACKUP_RESTORE: restore a backup chain into a new lvol - FN_BACKUP_MERGE: merge two backups to shorten the chain + +All three are multi-cycle: a task issues its RPC, defers, and polls the data +plane's transfer state on later cycles until it reaches a terminal state. """ import time -from simplyblock_core import constants, db_controller, utils -from simplyblock_core.controllers import backup_events, tasks_controller +from simplyblock_core import db_controller, utils +from simplyblock_core.controllers import backup_events from simplyblock_core.models.backup import Backup from simplyblock_core.models.cluster import Cluster from simplyblock_core.models.job_schedule import JobSchedule +from simplyblock_core.models.lvol_model import LVol from simplyblock_core.models.storage_node import StorageNode from simplyblock_core.rpc_client import RPCException +from simplyblock_core.services.task_runner_base import ( + RunnerSpec, + TaskAbort, + TaskDefer, + TaskRetry, + serve, +) logger = utils.get_logger(__name__) db = db_controller.DBController() +# Time-based backstop for a task that is stuck but not erroring. +_DEFAULT_BACKUP_TIMEOUT_SEC = 14400 + + +def _online_node(node_id): + """The node the task's RPCs go to, or a signal to stop/wait.""" + try: + snode = db.get_storage_node_by_id(node_id) + except KeyError: + raise TaskAbort(f"Node {node_id} not found") + + if snode.status != StorageNode.STATUS_ONLINE: + raise TaskRetry(f"Node {snode.status}, retrying") + return snode + + +def _transfer_state(rpc_client, bdev_name): + try: + stat = rpc_client.bdev_lvol_transfer_stat(bdev_name) + except RPCException: + raise TaskRetry("transfer stat RPC failed, retrying") -def _fail_backup(backup, task, message): - backup.status = Backup.STATUS_FAILED - backup.error_message = message - backup.write_to_db() - backup_events.backup_failed(backup.cluster_id, backup.node_id, backup) - task.function_result = message - task.status = JobSchedule.STATUS_DONE - task.write_to_db(db.kv_store) + if not stat or not isinstance(stat, dict): + raise TaskRetry("unexpected transfer stat response, retrying") + return stat.get("transfer_state", "") def _run_backup(task): backup_id = task.function_params.get("backup_id") if not backup_id: - task.function_result = "Missing backup_id" - task.status = JobSchedule.STATUS_DONE - task.write_to_db(db.kv_store) - return + raise TaskAbort("Missing backup_id") try: backup = db.get_backup_by_id(backup_id) except KeyError: - task.function_result = f"Backup {backup_id} not found" - task.status = JobSchedule.STATUS_DONE - task.write_to_db(db.kv_store) - return + raise TaskAbort(f"Backup {backup_id} not found") if backup.status not in (Backup.STATUS_PENDING, Backup.STATUS_IN_PROGRESS): - task.status = JobSchedule.STATUS_DONE - task.write_to_db(db.kv_store) - return - - try: - snode = db.get_storage_node_by_id(backup.node_id) - except KeyError: - _fail_backup(backup, task, f"Node {backup.node_id} not found") - return - - if snode.status != StorageNode.STATUS_ONLINE: - task.retry += 1 - task.function_result = f"Node {snode.status}, retrying" - task.status = JobSchedule.STATUS_SUSPENDED - task.write_to_db(db.kv_store) - return + raise TaskAbort(f"Backup is already {backup.status}") + snode = _online_node(backup.node_id) rpc_client = snode.rpc_client(timeout=30) - # Resolve snapshot bdev name (needed for both kick-off and polling) try: snapshot = db.get_snapshot_by_id(backup.snapshot_id) except KeyError: - _fail_backup(backup, task, f"Snapshot {backup.snapshot_id} not found") - return + raise TaskAbort(f"Snapshot {backup.snapshot_id} not found") snap_bdev_name = snapshot.snap_bdev if not snap_bdev_name: @@ -82,69 +87,43 @@ def _run_backup(task): if backup.status == Backup.STATUS_PENDING: try: ret = rpc_client.bdev_lvol_s3_backup(backup.s3_id, [snap_bdev_name], cluster_batch=16) - if not ret: - _fail_backup(backup, task, "bdev_lvol_s3_backup RPC failed") - return except RPCException as e: - _fail_backup(backup, task, f"RPC error: {e}") - return + raise TaskAbort(f"RPC error: {e}") + if not ret: + raise TaskAbort("bdev_lvol_s3_backup RPC failed") backup.status = Backup.STATUS_IN_PROGRESS backup.write_to_db() # Give the data plane time to start the transfer before polling - task.status = JobSchedule.STATUS_SUSPENDED - task.function_result = "Backup in progress" - task.write_to_db(db.kv_store) - return + raise TaskDefer("Backup in progress") - # Poll via bdev_lvol_transfer_stat on the snapshot bdev - try: - stat = rpc_client.bdev_lvol_transfer_stat(snap_bdev_name) - except RPCException: - task.retry += 1 - task.status = JobSchedule.STATUS_SUSPENDED - task.write_to_db(db.kv_store) + state = _transfer_state(rpc_client, snap_bdev_name) + if state == "Done": + backup.status = Backup.STATUS_COMPLETED + backup.completed_at = int(time.time()) + backup.write_to_db() + backup_events.backup_completed(backup.cluster_id, backup.node_id, backup) + task.function_result = "Backup completed" return - if stat and isinstance(stat, dict): - state = stat.get("transfer_state", "") - if state == "Done": - backup.status = Backup.STATUS_COMPLETED - backup.completed_at = int(time.time()) - backup.write_to_db() - backup_events.backup_completed(backup.cluster_id, backup.node_id, backup) - task.function_result = "Backup completed" - task.status = JobSchedule.STATUS_DONE - task.write_to_db(db.kv_store) - elif state == "Failed": - _fail_backup(backup, task, "Backup transfer failed on data plane") - elif state == "No process" and backup.status == Backup.STATUS_IN_PROGRESS: - # "No process" means no transfer is running for this bdev — the - # backup died (e.g. an SPDK crash wiped the in-flight transfer). - # Re-issue by resetting to PENDING, but COUNT it as a retry so the - # max_retry ceiling in process_task() can stop a backup that keeps - # failing. Without the increment this branch loops forever, and - # re-issuing an RPC that crashes the data plane just re-crashes it. - # NOTE: this treats "No process" as a failure. It relies on a - # healthy in-progress backup NOT sitting in "No process"; if the - # data plane ever reports "No process" for a running backup, this - # would fail it prematurely and completion needs another signal. - task.retry += 1 - backup.status = Backup.STATUS_PENDING - backup.write_to_db() - task.function_result = "No process, retrying backup start" - task.status = JobSchedule.STATUS_SUSPENDED - task.write_to_db(db.kv_store) - else: - # "In progress" — still running, retry later - task.status = JobSchedule.STATUS_SUSPENDED - task.function_result = "Backup in progress" - task.write_to_db(db.kv_store) - else: - # Unexpected response — retry - task.retry += 1 - task.status = JobSchedule.STATUS_SUSPENDED - task.write_to_db(db.kv_store) + if state == "Failed": + raise TaskAbort("Backup transfer failed on data plane") + + if state == "No process" and backup.status == Backup.STATUS_IN_PROGRESS: + # "No process" means no transfer is running for this bdev — the backup + # died (e.g. an SPDK crash wiped the in-flight transfer). Re-issue by + # resetting to PENDING, but COUNT it as a retry so the max_retry ceiling + # can stop a backup that keeps failing. Without that, re-issuing an RPC + # that crashes the data plane just re-crashes it, forever. + # NOTE: this treats "No process" as a failure. It relies on a healthy + # in-progress backup NOT sitting in "No process"; if the data plane ever + # reports "No process" for a running backup, this would fail it + # prematurely and completion needs another signal. + backup.status = Backup.STATUS_PENDING + backup.write_to_db() + raise TaskRetry("No process, retrying backup start") + + raise TaskDefer("Backup in progress") def _set_lvol_online(task): @@ -153,7 +132,6 @@ def _set_lvol_online(task): if not lvol_id: return try: - from simplyblock_core.models.lvol_model import LVol lvol = db.get_lvol_by_id(lvol_id) if lvol.status == LVol.STATUS_RESTORING: lvol.status = LVol.STATUS_ONLINE @@ -169,7 +147,6 @@ def _set_lvol_restore_failed(task, reason): if not lvol_id: return try: - from simplyblock_core.models.lvol_model import LVol lvol = db.get_lvol_by_id(lvol_id) if lvol.status == LVol.STATUS_RESTORING: lvol.status = LVol.STATUS_RESTORE_FAILED @@ -184,171 +161,96 @@ def _run_restore(task): lvol_name = task.function_params.get("lvol_name") chain_ids = task.function_params.get("chain_ids", []) node_id = task.node_id - recovery_started = task.function_params.get("recovery_started", False) - - try: - snode = db.get_storage_node_by_id(node_id) - except KeyError: - task.function_result = f"Node {node_id} not found" - task.status = JobSchedule.STATUS_DONE - task.write_to_db(db.kv_store) - return - - if snode.status != StorageNode.STATUS_ONLINE: - task.retry += 1 - task.status = JobSchedule.STATUS_SUSPENDED - task.write_to_db(db.kv_store) - return + snode = _online_node(node_id) rpc_client = snode.rpc_client(timeout=30) # Check that the target lvol still exists in DB before doing any RPC work lvol_id = task.function_params.get("lvol_id") if lvol_id: try: - from simplyblock_core.models.lvol_model import LVol - lvol = db.get_lvol_by_id(lvol_id) - if lvol.status == LVol.STATUS_IN_DELETION: - task.function_result = f"Restore target {lvol_id} has been deleted" - task.status = JobSchedule.STATUS_DONE - task.write_to_db(db.kv_store) - return + if db.get_lvol_by_id(lvol_id).status == LVol.STATUS_IN_DELETION: + raise TaskAbort(f"Restore target {lvol_id} has been deleted") except KeyError: - task.function_result = f"Restore target {lvol_id} no longer exists" - task.status = JobSchedule.STATUS_DONE - task.write_to_db(db.kv_store) - return + raise TaskAbort(f"Restore target {lvol_id} no longer exists") - if not recovery_started: + if not task.function_params.get("recovery_started", False): try: ret = rpc_client.bdev_lvol_s3_recovery(lvol_name, chain_ids, cluster_batch=16) - if not ret: - task.function_result = "bdev_lvol_s3_recovery RPC failed" - task.retry += 1 - task.status = JobSchedule.STATUS_SUSPENDED - task.write_to_db(db.kv_store) - return except RPCException as e: - task.function_result = f"RPC error: {e}" - task.retry += 1 - task.status = JobSchedule.STATUS_SUSPENDED - task.write_to_db(db.kv_store) - return + raise TaskRetry(f"RPC error: {e}") + if not ret: + raise TaskRetry("bdev_lvol_s3_recovery RPC failed") - # Mark recovery as started so we don't re-issue the RPC on subsequent polls + # Don't re-issue the RPC on subsequent polls, and give the data plane + # time to start the transfer before the first one. task.function_params["recovery_started"] = True - # Give the data plane time to start the transfer before polling - task.status = JobSchedule.STATUS_SUSPENDED - task.write_to_db(db.kv_store) - return + raise TaskDefer("Restore started") - # Poll via bdev_lvol_transfer_stat on the target lvol - try: - stat = rpc_client.bdev_lvol_transfer_stat(lvol_name) - except RPCException: - task.retry += 1 - task.status = JobSchedule.STATUS_SUSPENDED - task.write_to_db(db.kv_store) + state = _transfer_state(rpc_client, lvol_name) + if state == "Done": + _set_lvol_online(task) + try: + backup = db.get_backup_by_id(backup_id) + backup_events.backup_restore_completed( + task.cluster_id, node_id, backup, lvol_name) + except KeyError: + logger.warning( + f"Backup {backup_id} no longer exists, " + f"skipping restore-completed event for {lvol_name}") + task.function_result = f"Restore completed: {lvol_name}" return - if stat and isinstance(stat, dict): - state = stat.get("transfer_state", "") - if state == "Done": - _set_lvol_online(task) - try: - backup = db.get_backup_by_id(backup_id) - backup_events.backup_restore_completed( - task.cluster_id, node_id, backup, lvol_name) - except KeyError: - pass - task.function_result = f"Restore completed: {lvol_name}" - task.status = JobSchedule.STATUS_DONE - task.write_to_db(db.kv_store) - elif state == "Failed": - fail_count = task.function_params.get("fail_count", 0) + 1 - task.function_params["fail_count"] = fail_count - reason = f"S3 transfer failed on data plane (attempt {fail_count})" - task.function_result = reason - if fail_count >= 3: - _set_lvol_restore_failed(task, reason) - try: - backup = db.get_backup_by_id(backup_id) - backup_events.backup_restore_failed( - task.cluster_id, node_id, backup, lvol_name, reason) - except KeyError: - logger.warning( - "Backup %s not found in DB; restore-failed event skipped for lvol %s", - backup_id, lvol_name) - task.status = JobSchedule.STATUS_DONE - else: - task.retry += 1 - task.status = JobSchedule.STATUS_SUSPENDED - task.write_to_db(db.kv_store) - elif state == "No process": - task.function_params["recovery_started"] = False - task.status = JobSchedule.STATUS_SUSPENDED - task.write_to_db(db.kv_store) - else: - # "In progress" — still running, retry later - task.status = JobSchedule.STATUS_SUSPENDED - task.write_to_db(db.kv_store) - else: - # Unexpected response — retry - task.retry += 1 - task.status = JobSchedule.STATUS_SUSPENDED - task.write_to_db(db.kv_store) + if state == "Failed": + fail_count = task.function_params.get("fail_count", 0) + 1 + task.function_params["fail_count"] = fail_count + reason = f"S3 transfer failed on data plane (attempt {fail_count})" + if fail_count < 3: + raise TaskRetry(reason) + + _set_lvol_restore_failed(task, reason) + try: + backup = db.get_backup_by_id(backup_id) + backup_events.backup_restore_failed( + task.cluster_id, node_id, backup, lvol_name, reason) + except KeyError: + logger.warning( + "Backup %s not found in DB; restore-failed event skipped for lvol %s", + backup_id, lvol_name) + raise TaskAbort(reason) + + if state == "No process": + task.function_params["recovery_started"] = False + raise TaskDefer("No process, restarting recovery") + + raise TaskDefer("Restore in progress") def _run_merge(task): keep_backup_id = task.function_params.get("keep_backup_id") old_backup_id = task.function_params.get("old_backup_id") - merge_started = task.function_params.get("merge_started", False) try: keep_backup = db.get_backup_by_id(keep_backup_id) old_backup = db.get_backup_by_id(old_backup_id) except KeyError as e: - task.function_result = str(e) - task.status = JobSchedule.STATUS_DONE - task.write_to_db(db.kv_store) - return - - try: - snode = db.get_storage_node_by_id(keep_backup.node_id) - except KeyError: - task.function_result = f"Node {keep_backup.node_id} not found" - task.status = JobSchedule.STATUS_DONE - task.write_to_db(db.kv_store) - return - - if snode.status != StorageNode.STATUS_ONLINE: - task.retry += 1 - task.status = JobSchedule.STATUS_SUSPENDED - task.write_to_db(db.kv_store) - return + raise TaskAbort(str(e)) + snode = _online_node(keep_backup.node_id) rpc_client = snode.rpc_client(timeout=30) - if not merge_started: + if not task.function_params.get("merge_started", False): try: - ret = rpc_client.bdev_lvol_s3_merge(keep_backup.s3_id, old_backup.s3_id, cluster_batch=16, lvs_name=snode.lvstore) - if not ret: - task.function_result = "bdev_lvol_s3_merge RPC failed" - task.retry += 1 - task.status = JobSchedule.STATUS_SUSPENDED - task.write_to_db(db.kv_store) - return + ret = rpc_client.bdev_lvol_s3_merge( + keep_backup.s3_id, old_backup.s3_id, cluster_batch=16, lvs_name=snode.lvstore) except RPCException as e: - task.function_result = f"RPC error: {e}" - task.retry += 1 - task.status = JobSchedule.STATUS_SUSPENDED - task.write_to_db(db.kv_store) - return + raise TaskRetry(f"RPC error: {e}") + if not ret: + raise TaskRetry("bdev_lvol_s3_merge RPC failed") task.function_params["merge_started"] = True - task.write_to_db(db.kv_store) # Give the data plane time to complete the merge before finalizing - return + raise TaskDefer("Merge started") # The merge RPC is synchronous on the data plane — once it returned # successfully, the S3 data has been merged. Finalize: update the @@ -361,116 +263,79 @@ def _run_merge(task): old_backup.write_to_db() task.function_result = "Merge completed" - task.status = JobSchedule.STATUS_DONE - task.write_to_db(db.kv_store) logger.info(f"Merge completed: {old_backup_id} merged into {keep_backup_id}") -def _terminate_task(task, reason): - """Terminate a backup/restore/merge task and finalize its resource. +_HANDLERS = { + JobSchedule.FN_BACKUP: _run_backup, + JobSchedule.FN_BACKUP_RESTORE: _run_restore, + JobSchedule.FN_BACKUP_MERGE: _run_merge, +} + + +def process_task(task): + cluster = db.get_cluster_by_id(task.cluster_id) + backup_timeout_sec = getattr(cluster, 'backup_timeout_seconds', 0) or _DEFAULT_BACKUP_TIMEOUT_SEC + elapsed = int(time.time()) - task.date if task.date else 0 + if elapsed > backup_timeout_sec: + raise TaskAbort(f"timeout after {elapsed}s") + + _HANDLERS[task.function_name](task) - Shared by the time-based timeout and the max_retry ceiling so both stop - the task the same way instead of leaving it to loop. + +def finalize_resource(task): + """Release the backup/restore/merge the task was driving, once it is over. + + Reached on every terminal path, so it is written to be a no-op when the + handler completed the resource itself and to only act when the task ended + with the resource still in flight — a timeout, the retry ceiling, an abort + or a cancellation, none of which the handler sees. """ + reason = task.function_result + if task.function_name == JobSchedule.FN_BACKUP: - bid = task.function_params.get("backup_id") - if bid: - try: - b = db.get_backup_by_id(bid) - if b.status in (Backup.STATUS_PENDING, Backup.STATUS_IN_PROGRESS): - _fail_backup(b, task, reason) - return - except KeyError: - pass + backup_id = task.function_params.get("backup_id") + if not backup_id: + return + try: + backup = db.get_backup_by_id(backup_id) + except KeyError: + return + if backup.status in (Backup.STATUS_PENDING, Backup.STATUS_IN_PROGRESS): + backup.status = Backup.STATUS_FAILED + backup.error_message = reason + backup.write_to_db() + backup_events.backup_failed(backup.cluster_id, backup.node_id, backup) + elif task.function_name == JobSchedule.FN_BACKUP_RESTORE: _set_lvol_restore_failed(task, reason) + elif task.function_name == JobSchedule.FN_BACKUP_MERGE: - old_bid = task.function_params.get("old_backup_id") - if old_bid: - try: - ob = db.get_backup_by_id(old_bid) - if ob.status == Backup.STATUS_MERGING: - # Merge did not finish; leave the old backup intact. - ob.status = Backup.STATUS_COMPLETED - ob.write_to_db() - except KeyError: - pass - - task.function_result = reason - task.status = JobSchedule.STATUS_DONE - task.write_to_db(db.kv_store) - - -def process_task(task, cl): - """Advance a single backup task by one step, or terminate it. - - Terminates on cancellation, on the time-based timeout, or once the - max_retry ceiling is reached — the last is what stops a backup that keeps - crashing the data plane from re-issuing its RPC forever. - """ - if task.canceled: - task.function_result = "canceled" - task.status = JobSchedule.STATUS_DONE - task.write_to_db(db.kv_store) - return + old_backup_id = task.function_params.get("old_backup_id") + if not old_backup_id: + return + try: + old_backup = db.get_backup_by_id(old_backup_id) + except KeyError: + return + if old_backup.status == Backup.STATUS_MERGING: + # Merge did not finish; leave the old backup intact. + old_backup.status = Backup.STATUS_COMPLETED + old_backup.write_to_db() - # Time-based backstop for a task that is stuck but not erroring (default 4h). - backup_timeout_sec = getattr(cl, 'backup_timeout_seconds', 0) or 14400 - elapsed = int(time.time()) - task.date if task.date else 0 - if elapsed > backup_timeout_sec: - _terminate_task(task, f"timeout after {elapsed}s") - return - # Retry ceiling: every other task runner enforces this. Without it a task - # whose step keeps failing (e.g. an RPC that crashes SPDK) loops until the - # timeout, re-triggering the failure each cycle. max_retry < 0 means the - # task is intentionally unbounded and only the timeout applies. - if 0 <= task.max_retry <= task.retry: - _terminate_task(task, f"max retry reached ({task.retry}/{task.max_retry})") - return +SPEC = RunnerSpec( + name="tasks-runner-backup", + function_names=list(_HANDLERS), + handler=process_task, + on_finish=finalize_resource, + is_eligible=lambda task, cluster: cluster.status != Cluster.STATUS_IN_ACTIVATION, +) - try: - if task.function_name == JobSchedule.FN_BACKUP: - _run_backup(task) - elif task.function_name == JobSchedule.FN_BACKUP_RESTORE: - _run_restore(task) - elif task.function_name == JobSchedule.FN_BACKUP_MERGE: - _run_merge(task) - except Exception as e: - logger.error(f"Error running backup task {task.uuid}: {e}") - # Increment retry so the task eventually reaches max_retry - # instead of looping forever on non-RPCException errors - task.retry += 1 - task.function_result = f"Unhandled error: {e}" - task.status = JobSchedule.STATUS_SUSPENDED - task.write_to_db(db.kv_store) + +def main(): + serve(SPEC) if __name__ == "__main__": - logger.info("Starting backup tasks runner...") - while True: - clusters = db.get_clusters() - for cl in clusters: - if cl.status == Cluster.STATUS_IN_ACTIVATION: - continue - - tasks = db.get_job_tasks(cl.get_id(), reverse=False) - for task in tasks: - if task.function_name not in ( - JobSchedule.FN_BACKUP, - JobSchedule.FN_BACKUP_RESTORE, - JobSchedule.FN_BACKUP_MERGE, - ): - continue - if task.status == JobSchedule.STATUS_DONE or task.canceled: - continue - - if not tasks_controller.claim_task(task): - logger.info(f"Backup task {task.uuid} owned by another runner host; skipping") - continue - - # Re-fetch task for freshness - task = db.get_task_by_id(task.uuid) - process_task(task, cl) - - time.sleep(constants.TASK_EXEC_INTERVAL_SEC) + main() diff --git a/simplyblock_core/services/tasks_runner_cluster_expand.py b/simplyblock_core/services/tasks_runner_cluster_expand.py index 03a3a4e6e6..70c4f39d96 100644 --- a/simplyblock_core/services/tasks_runner_cluster_expand.py +++ b/simplyblock_core/services/tasks_runner_cluster_expand.py @@ -1,8 +1,5 @@ # coding=utf-8 -import time - - -from simplyblock_core import db_controller, utils, constants +from simplyblock_core import db_controller, utils from simplyblock_core.controllers import tasks_controller from simplyblock_core.controllers.cluster_expansion.executor import ( integrate_new_node_into_cluster, @@ -14,6 +11,12 @@ ) from simplyblock_core.models.job_schedule import JobSchedule from simplyblock_core.models.nvme_device import NVMeDevice +from simplyblock_core.services.task_runner_base import ( + RunnerSpec, + TaskAbort, + TaskRetry, + serve, +) logger = utils.get_logger(__name__) @@ -23,127 +26,54 @@ def process_task(task): - if task.canceled: - task.function_result = "canceled" - task.status = JobSchedule.STATUS_DONE - task.write_to_db(db.kv_store) - return False - - # max_retry < 0 means never give up (the plan must complete — an - # abandoned half-moved topology strands stale sec/tert pointers); - # cancel the task to stop it deliberately. - if 0 <= task.max_retry <= task.retry: - task.function_result = "max retry reached" - task.status = JobSchedule.STATUS_DONE - task.write_to_db(db.kv_store) - return True - new_node_id = task.function_params.get("new_node_id") if not new_node_id: - task.function_result = "missing new_node_id in function_params" - task.status = JobSchedule.STATUS_DONE - task.write_to_db(db.kv_store) - return True - - if task.status != JobSchedule.STATUS_RUNNING: - task.status = JobSchedule.STATUS_RUNNING - task.write_to_db(db.kv_store) - - try: - cluster = db.get_cluster_by_id(task.cluster_id) - new_snode = db.get_storage_node_by_id(new_node_id) - - # Retry-by-resume: a prior attempt that aborted left expand_state at - # the failed move's cursor. Flip it back to in_progress so the - # orchestrator re-attempts that move instead of recomputing a fresh - # diff against a topology the aborted run may have partially mutated. - if (cluster.expand_state or {}).get("phase") == EXPAND_PHASE_ABORTED: - cluster.expand_state = expand_state_rearm(cluster.expand_state) - cluster.write_to_db() - - integrate_new_node_into_cluster( - cluster, new_snode, db_controller=db, - manage_cluster_status=True) - - # integrate_new_node_into_cluster returns only on success; the - # orchestrator marks expand_state completed. Re-read to confirm. - cluster = db.get_cluster_by_id(task.cluster_id) - phase = (cluster.expand_state or {}).get("phase") - if phase == EXPAND_PHASE_COMPLETED: - # Queue new-device migration now that the rotation has landed and - # the cluster is back to ACTIVE — tasks are created against the - # post-rotation lvstore_stack (which includes the newcomer's - # primary distr). Mirrors the trigger the non-expansion add path - # runs inside add_node. - new_snode = db.get_storage_node_by_id(new_node_id) - for dev in new_snode.nvme_devices: - if dev.status == NVMeDevice.STATUS_ONLINE: - tasks_controller.add_new_device_mig_task(dev.get_id()) - task.function_result = f"expansion complete: {new_node_id}" - task.status = JobSchedule.STATUS_DONE - else: - # Shouldn't happen (no exception but not completed) — suspend and - # retry rather than silently mark done. - task.function_result = f"unexpected phase after run: {phase!r}" - task.retry += 1 - task.status = JobSchedule.STATUS_SUSPENDED - task.write_to_db(db.kv_store) - return True - except Exception as e: - logger.error(e) - task.function_result = f"attempt {task.retry + 1} failed: {e}" - task.retry += 1 - task.status = JobSchedule.STATUS_SUSPENDED - task.write_to_db(db.kv_store) - return False + raise TaskAbort("missing new_node_id in function_params") + + cluster = db.get_cluster_by_id(task.cluster_id) + new_snode = db.get_storage_node_by_id(new_node_id) + + # Retry-by-resume: a prior attempt that aborted left expand_state at + # the failed move's cursor. Flip it back to in_progress so the + # orchestrator re-attempts that move instead of recomputing a fresh + # diff against a topology the aborted run may have partially mutated. + if (cluster.expand_state or {}).get("phase") == EXPAND_PHASE_ABORTED: + cluster.expand_state = expand_state_rearm(cluster.expand_state) + cluster.write_to_db() + + integrate_new_node_into_cluster( + cluster, new_snode, db_controller=db, + manage_cluster_status=True) + + # integrate_new_node_into_cluster returns only on success; the + # orchestrator marks expand_state completed. Re-read to confirm. + cluster = db.get_cluster_by_id(task.cluster_id) + phase = (cluster.expand_state or {}).get("phase") + if phase != EXPAND_PHASE_COMPLETED: + raise TaskRetry(f"unexpected phase after run: {phase!r}") + + # Queue new-device migration now that the rotation has landed and + # the cluster is back to ACTIVE — tasks are created against the + # post-rotation lvstore_stack (which includes the newcomer's + # primary distr). Mirrors the trigger the non-expansion add path + # runs inside add_node. + new_snode = db.get_storage_node_by_id(new_node_id) + for dev in new_snode.nvme_devices: + if dev.status == NVMeDevice.STATUS_ONLINE: + tasks_controller.add_new_device_mig_task(dev.get_id()) + + task.function_result = f"expansion complete: {new_node_id}" + + +SPEC = RunnerSpec( + name="tasks-runner-cluster-expand", + function_names=[JobSchedule.FN_CLUSTER_EXPAND], + handler=process_task, +) def main(): - logger.info("Starting Tasks runner cluster expand...") - while True: - clusters = db.get_clusters() - if not clusters: - logger.error("No clusters found!") - else: - for cl in clusters: - tasks = db.get_job_tasks(cl.get_id(), reverse=False) - for task in tasks: - delay_seconds = constants.TASK_EXEC_INTERVAL_SEC - if task.function_name != JobSchedule.FN_CLUSTER_EXPAND: - continue - # Per-task isolation: a crash in process_task must not - # escape to the outer loop and kill the runner. - try: - while task.status != JobSchedule.STATUS_DONE: - # Re-fetch: the task may have been cancelled. - task = db.get_task_by_id(task.uuid) - # Lease gate: skip a task another live runner owns. - if not tasks_controller.claim_task(task): - logger.info( - f"Cluster-expand task {task.uuid} owned by " - f"another runner host; skipping") - break - with tasks_controller.task_lease_heartbeat(task): - res = process_task(task) - if res: - if task.status == JobSchedule.STATUS_DONE: - break - else: - # Cap the exponential backoff so a permanently - # failing expansion can't grow the sleep without - # bound. - delay_seconds = min( - delay_seconds * 2, - constants.RESTART_TASK_EXEC_INTERVAL_MAX_SEC, - ) - time.sleep(delay_seconds) - except Exception as e: - logger.error( - f"Cluster-expand task {task.uuid} processing " - f"crashed: {e}") - logger.exception(e) - - time.sleep(constants.TASK_EXEC_INTERVAL_SEC) + serve(SPEC) if __name__ == "__main__": diff --git a/tests/integration/test_tasks_runner_cluster_expand.py b/tests/integration/test_tasks_runner_cluster_expand.py index 7fae28f68c..98e70afa1c 100644 --- a/tests/integration/test_tasks_runner_cluster_expand.py +++ b/tests/integration/test_tasks_runner_cluster_expand.py @@ -4,6 +4,10 @@ No FDB / SPDK: ``integrate_new_node_into_cluster`` and the DB handle are mocked, so these run in milliseconds. This is the "fast tier" that lets expansion logic be developed without the multi-hour real-FDB simulation. + +Scope is the handler only. Cancellation, max-retry, status transitions and +the retry counter belong to the task runner driver and are tested once for +every runner in ``tests/unit/tasks/test_task_runner_base.py``. """ import unittest @@ -16,6 +20,7 @@ EXPAND_PHASE_COMPLETED, EXPAND_PHASE_IN_PROGRESS, ) +from simplyblock_core.services.task_runner_base import TaskAbort, TaskRetry import simplyblock_core.services.tasks_runner_cluster_expand as runner @@ -65,27 +70,10 @@ def setUp(self): self.tc = patch.object(runner, "tasks_controller").start() - def test_canceled_marks_done(self): - task = _task(canceled=True) - res = runner.process_task(task) - self.assertFalse(res) - self.assertEqual(task.status, JobSchedule.STATUS_DONE) - self.assertEqual(task.function_result, "canceled") - self.integrate.assert_not_called() - - def test_max_retry_marks_done(self): - task = _task(retry=3, max_retry=3) - res = runner.process_task(task) - self.assertTrue(res) - self.assertEqual(task.status, JobSchedule.STATUS_DONE) - self.assertEqual(task.function_result, "max retry reached") - self.integrate.assert_not_called() - - def test_missing_new_node_id_marks_done(self): + def test_missing_new_node_id_aborts(self): task = _task(new_node_id=None) - res = runner.process_task(task) - self.assertTrue(res) - self.assertEqual(task.status, JobSchedule.STATUS_DONE) + with self.assertRaises(TaskAbort): + runner.process_task(task) self.integrate.assert_not_called() def test_happy_path_completes_and_queues_dev_mig(self): @@ -101,26 +89,34 @@ def _integrate(c, snode, **kw): self.integrate.side_effect = _integrate task = _task() - res = runner.process_task(task) + runner.process_task(task) - self.assertTrue(res) - self.assertEqual(task.status, JobSchedule.STATUS_DONE) self.integrate.assert_called_once() + self.assertIn("expansion complete", task.function_result) # Only the two ONLINE devices get a migration task. self.assertEqual(self.tc.add_new_device_mig_task.call_count, 2) - def test_failure_suspends_and_increments_retry(self): + def test_failure_propagates_for_the_driver_to_retry(self): self.db.get_cluster_by_id.return_value = _cluster() self.db.get_storage_node_by_id.return_value = _node_with_devices() self.integrate.side_effect = RuntimeError("boom") task = _task(retry=0) - res = runner.process_task(task) + # Suspending and counting the retry is the driver's half of the + # contract; the handler only has to not swallow the failure. + with self.assertRaises(RuntimeError): + runner.process_task(task) + + self.tc.add_new_device_mig_task.assert_not_called() + + def test_unexpected_phase_after_run_is_retried(self): + self.db.get_cluster_by_id.return_value = _cluster() + self.db.get_storage_node_by_id.return_value = _node_with_devices() + self.integrate.side_effect = lambda c, snode, **kw: None + + with self.assertRaises(TaskRetry): + runner.process_task(_task()) - self.assertFalse(res) - self.assertEqual(task.status, JobSchedule.STATUS_SUSPENDED) - self.assertEqual(task.retry, 1) - self.assertIn("boom", task.function_result) self.tc.add_new_device_mig_task.assert_not_called() def test_aborted_state_is_rearmed_before_resume(self): diff --git a/tests/unit/tasks/test_retry_ceiling.py b/tests/unit/tasks/test_retry_ceiling.py index d99c490acf..8d2fa00a3f 100644 --- a/tests/unit/tasks/test_retry_ceiling.py +++ b/tests/unit/tasks/test_retry_ceiling.py @@ -32,122 +32,12 @@ import pytest -from simplyblock_core.models.backup import Backup from simplyblock_core.models.cluster import Cluster from simplyblock_core.models.job_schedule import JobSchedule from simplyblock_core.models.lvol_migration_group import LVolMigrationGroup from simplyblock_core.models.storage_node import StorageNode -# -------------------------------------------------------------------------- -# Backup runner: terminates instead of looping forever. -# -# The backup runner's loop is inline under ``if __name__ == '__main__'`` (it has -# no ``def main()``), so it is driven at the ``process_task`` level here. The -# other runners below are driven through their real ``main()``. -# -------------------------------------------------------------------------- - -@pytest.fixture -def runner(monkeypatch): - """Import the backup runner with its FDB writes and events neutralised. - - The module's ``while True`` loop is guarded by ``if __name__ == - '__main__'``, so importing it only defines functions and a (kv_store=None) - ``DBController`` singleton — safe under the unit tier's stubbed ``fdb``. - """ - import simplyblock_core.services.tasks_runner_backup as runner - monkeypatch.setattr(runner, "backup_events", MagicMock()) - # Model writes would otherwise dereference the None kv_store and exit(1). - monkeypatch.setattr(Backup, "write_to_db", MagicMock()) - monkeypatch.setattr(JobSchedule, "write_to_db", MagicMock()) - return runner - - -def _backup_task(retry, max_retry): - task = JobSchedule() - task.uuid = "task-1" - task.function_name = JobSchedule.FN_BACKUP - task.function_params = {"backup_id": "bk-1"} - task.retry = retry - task.max_retry = max_retry - task.canceled = False - # Recent enough that the 4h time-based timeout does not fire first. - task.date = int(__import__("time").time()) - return task - - -def _cluster(): - cl = MagicMock() - cl.backup_timeout_seconds = 14400 - return cl - - -def test_backup_fails_when_max_retry_reached(runner, monkeypatch): - """retry >= max_retry must fail the backup and finish the task, not poll.""" - backup = Backup() - backup.uuid = "bk-1" - backup.status = Backup.STATUS_IN_PROGRESS - - fake_db = MagicMock() - fake_db.get_backup_by_id.return_value = backup - monkeypatch.setattr(runner, "db", fake_db) - # If the ceiling is missing this would run and (in the incident) re-issue - # the crash-inducing RPC; assert it is never reached. - run_backup = MagicMock() - monkeypatch.setattr(runner, "_run_backup", run_backup) - - task = _backup_task(retry=10, max_retry=10) - runner.process_task(task, _cluster()) - - assert backup.status == Backup.STATUS_FAILED - assert task.status == JobSchedule.STATUS_DONE - assert "max retry" in task.function_result - run_backup.assert_not_called() - - -def test_backup_runs_below_max_retry(runner, monkeypatch): - """Below the ceiling the task still advances (dispatches its step).""" - monkeypatch.setattr(runner, "db", MagicMock()) - run_backup = MagicMock() - monkeypatch.setattr(runner, "_run_backup", run_backup) - - task = _backup_task(retry=9, max_retry=10) - runner.process_task(task, _cluster()) - - run_backup.assert_called_once_with(task) - - -def test_no_process_poll_counts_as_a_retry(runner, monkeypatch): - """The 'No process' re-issue branch must advance task.retry, otherwise the - ceiling can never bind to that path and the backup re-issues forever.""" - backup = Backup() - backup.uuid = "bk-1" - backup.status = Backup.STATUS_IN_PROGRESS - backup.snapshot_id = "snap-1" - - snode = MagicMock() - snode.status = StorageNode.STATUS_ONLINE - rpc = MagicMock() - rpc.bdev_lvol_transfer_stat.return_value = {"transfer_state": "No process"} - snode.rpc_client.return_value = rpc - - snapshot = MagicMock() - snapshot.snap_bdev = "lvs/snap0" - - fake_db = MagicMock() - fake_db.get_backup_by_id.return_value = backup - fake_db.get_storage_node_by_id.return_value = snode - fake_db.get_snapshot_by_id.return_value = snapshot - monkeypatch.setattr(runner, "db", fake_db) - - task = _backup_task(retry=3, max_retry=10) - runner._run_backup(task) - - assert task.retry == 4, "No-process re-issue must count toward the ceiling" - assert backup.status == Backup.STATUS_PENDING - rpc.bdev_lvol_s3_backup.assert_not_called() # this poll only resets state - - # -------------------------------------------------------------------------- # Drive-main harness: exercise each runner's real retry loop end-to-end. # -------------------------------------------------------------------------- @@ -302,16 +192,6 @@ def _wire_base(runner, monkeypatch, task): # tree, so a new retry-driven runner shows up as a failing case until a spec is # added here. -def _spec_cluster_expand(runner, monkeypatch): - task = _make_task(JobSchedule.FN_CLUSTER_EXPAND, new_node_id="new-1") - _wire_base(runner, monkeypatch, task) - # The actual expansion work fails every cycle. - monkeypatch.setattr( - runner, "integrate_new_node_into_cluster", - MagicMock(side_effect=RuntimeError("expand boom"))) - return task - - def _spec_node_add(runner, monkeypatch): task = _make_task(JobSchedule.FN_NODE_ADD) _wire_base(runner, monkeypatch, task) @@ -398,7 +278,6 @@ def _get_node(node_id): # name -> spec for the runners driven through their real main() loop. _MAIN_DRIVEN_SPECS = { - "tasks_runner_cluster_expand.py": _spec_cluster_expand, "tasks_runner_node_add.py": _spec_node_add, "tasks_runner_restart.py": _spec_restart, "tasks_runner_batch_migration.py": _spec_batch_migration, @@ -408,9 +287,7 @@ def _get_node(node_id): # generic drive-main harness. The backup runner's loop is inline under # ``if __name__ == '__main__'`` (no ``def main()``), so it is exercised at the # ``process_task`` level by ``test_backup_*`` above. -_COVERED_ELSEWHERE = { - "tasks_runner_backup.py": "driven via process_task in test_backup_* above", -} +_COVERED_ELSEWHERE: dict = {} # Runners migrated onto the shared driver (``task_runner_base``). They no longer # own a loop or a retry counter — the driver enforces the ceiling for all of @@ -424,6 +301,8 @@ def _get_node(node_id): "tasks_runner_jc_comp.py", "tasks_runner_replication_final.py", "tasks_runner_sync_lvol_del.py", + "tasks_runner_backup.py", + "tasks_runner_cluster_expand.py", } # Runners that increment task.retry but are intentionally UNBOUNDED: the diff --git a/tests/unit/tasks/test_runner_specs.py b/tests/unit/tasks/test_runner_specs.py index 84bbacf29d..34bcf85c6c 100644 --- a/tests/unit/tasks/test_runner_specs.py +++ b/tests/unit/tasks/test_runner_specs.py @@ -6,10 +6,12 @@ eligibility predicate. These tests pin that translation — the loop, lease and retry mechanics themselves are covered once in ``test_task_runner_base.py``. """ +import time from unittest.mock import MagicMock import pytest +from simplyblock_core.models.backup import Backup from simplyblock_core.models.cluster import Cluster from simplyblock_core.models.storage_node import StorageNode from simplyblock_core.models.job_schedule import JobSchedule @@ -168,3 +170,179 @@ def test_jc_comp_retries_when_resume_fails(jc_comp): with pytest.raises(trb.TaskRetry): jc_comp.SPEC.handler(_task()) + + +# -- backup ----------------------------------------------------------------- + +@pytest.fixture +def backup_runner(monkeypatch): + """The backup runner with its DB, events and model writes neutralised. + + Its ceiling regression (an S3 backup whose bdev_lvol_s3_backup crashed SPDK + was re-issued every poll forever, and the backup never transitioned to + failed) now rests on two things this exercises: the "No process" poll + raising TaskRetry so the ceiling can bind, and finalize_resource failing the + backup on whichever terminal path the task ends up taking. + """ + import simplyblock_core.services.tasks_runner_backup as runner + monkeypatch.setattr(runner, "db", MagicMock()) + monkeypatch.setattr(runner, "backup_events", MagicMock()) + monkeypatch.setattr(Backup, "write_to_db", MagicMock()) + runner.db.get_cluster_by_id.return_value.backup_timeout_seconds = 14400 + return runner + + +def _backup_task(function_name=JobSchedule.FN_BACKUP, **params): + task = _task(**params) + task.function_name = function_name + task.date = int(time.time()) + return task + + +def _backup(status=Backup.STATUS_IN_PROGRESS): + backup = Backup() + backup.uuid = "bk-1" + backup.status = status + backup.snapshot_id = "snap-1" + return backup + + +def test_backup_no_process_poll_is_a_retry(backup_runner): + """The re-issue branch must consume a retry — without it the ceiling can + never bind to this path and the backup re-issues forever.""" + backup = _backup() + backup_runner.db.get_backup_by_id.return_value = backup + + snode = _node() + rpc = snode.rpc_client.return_value + rpc.bdev_lvol_transfer_stat.return_value = {"transfer_state": "No process"} + backup_runner.db.get_storage_node_by_id.return_value = snode + + with pytest.raises(trb.TaskRetry): + backup_runner.SPEC.handler(_backup_task(backup_id="bk-1")) + + assert backup.status == Backup.STATUS_PENDING + rpc.bdev_lvol_s3_backup.assert_not_called() # this poll only resets state + + +def test_backup_completes_on_done(backup_runner): + backup = _backup() + backup_runner.db.get_backup_by_id.return_value = backup + snode = _node() + snode.rpc_client.return_value.bdev_lvol_transfer_stat.return_value = { + "transfer_state": "Done"} + backup_runner.db.get_storage_node_by_id.return_value = snode + + task = _backup_task(backup_id="bk-1") + assert backup_runner.SPEC.handler(task) is None + assert backup.status == Backup.STATUS_COMPLETED + assert task.function_result == "Backup completed" + + +def test_backup_times_out(backup_runner): + backup_runner.db.get_cluster_by_id.return_value = _cluster() + backup_runner.db.get_cluster_by_id.return_value.backup_timeout_seconds = 10 + + task = _backup_task(backup_id="bk-1") + task.date = int(time.time()) - 3600 + + with pytest.raises(trb.TaskAbort, match="timeout"): + backup_runner.SPEC.handler(task) + + +def test_unfinished_backup_is_failed_when_the_task_ends(backup_runner): + """Whichever way the task ended — ceiling, timeout, abort, cancellation — + the backup must not be left sitting in a pending state forever.""" + backup = _backup() + backup_runner.db.get_backup_by_id.return_value = backup + + task = _backup_task(backup_id="bk-1") + task.function_result = "max retry reached (10/10)" + backup_runner.SPEC.on_finish(task) + + assert backup.status == Backup.STATUS_FAILED + assert backup.error_message == "max retry reached (10/10)" + backup_runner.backup_events.backup_failed.assert_called_once() + + +def test_completed_backup_survives_the_task_ending(backup_runner): + backup = _backup(Backup.STATUS_COMPLETED) + backup_runner.db.get_backup_by_id.return_value = backup + + backup_runner.SPEC.on_finish(_backup_task(backup_id="bk-1")) + + assert backup.status == Backup.STATUS_COMPLETED + backup_runner.backup_events.backup_failed.assert_not_called() + + +def test_unfinished_merge_leaves_the_old_backup_intact(backup_runner): + old = _backup(Backup.STATUS_MERGING) + backup_runner.db.get_backup_by_id.return_value = old + + backup_runner.SPEC.on_finish( + _backup_task(JobSchedule.FN_BACKUP_MERGE, old_backup_id="bk-0")) + + assert old.status == Backup.STATUS_COMPLETED + + +# -- cluster_expand --------------------------------------------------------- + +@pytest.fixture +def cluster_expand(monkeypatch): + import simplyblock_core.services.tasks_runner_cluster_expand as runner + monkeypatch.setattr(runner, "db", MagicMock()) + monkeypatch.setattr(runner, "tasks_controller", MagicMock()) + monkeypatch.setattr(runner, "integrate_new_node_into_cluster", MagicMock()) + return runner + + +def _expanded_cluster(runner, phase): + cluster = MagicMock() + cluster.expand_state = {"phase": phase} + runner.db.get_cluster_by_id.return_value = cluster + return cluster + + +def test_cluster_expand_completes_and_queues_device_migration(cluster_expand): + from simplyblock_core.controllers.cluster_expansion.planner import EXPAND_PHASE_COMPLETED + from simplyblock_core.models.nvme_device import NVMeDevice + + _expanded_cluster(cluster_expand, EXPAND_PHASE_COMPLETED) + device = MagicMock() + device.status = NVMeDevice.STATUS_ONLINE + device.get_id.return_value = "dev-1" + cluster_expand.db.get_storage_node_by_id.return_value.nvme_devices = [device] + + task = _task(new_node_id="new-1") + assert cluster_expand.SPEC.handler(task) is None + cluster_expand.tasks_controller.add_new_device_mig_task.assert_called_once_with("dev-1") + assert task.function_result == "expansion complete: new-1" + + +def test_cluster_expand_rearms_an_aborted_plan(cluster_expand): + from simplyblock_core.controllers.cluster_expansion.planner import ( + EXPAND_PHASE_ABORTED, + EXPAND_PHASE_COMPLETED, + ) + + cluster = MagicMock() + cluster.expand_state = {"phase": EXPAND_PHASE_ABORTED} + completed = MagicMock() + completed.expand_state = {"phase": EXPAND_PHASE_COMPLETED} + cluster_expand.db.get_cluster_by_id.side_effect = [cluster, completed] + cluster_expand.db.get_storage_node_by_id.return_value.nvme_devices = [] + + cluster_expand.SPEC.handler(_task(new_node_id="new-1")) + assert cluster.expand_state["phase"] != EXPAND_PHASE_ABORTED + cluster.write_to_db.assert_called_once() + + +def test_cluster_expand_retries_an_incomplete_phase(cluster_expand): + _expanded_cluster(cluster_expand, "in_progress") + with pytest.raises(trb.TaskRetry, match="unexpected phase"): + cluster_expand.SPEC.handler(_task(new_node_id="new-1")) + + +def test_cluster_expand_aborts_without_a_node(cluster_expand): + with pytest.raises(trb.TaskAbort): + cluster_expand.SPEC.handler(_task()) From dcd1fa4f6eab87c1302c7f6ce3141a61a97fade5 Mon Sep 17 00:00:00 2001 From: Max Schettler Date: Wed, 5 Aug 2026 13:40:45 +0200 Subject: [PATCH 13/25] Migrate node add runner onto the task runner MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The hand-rolled thread pool, in-flight sets and per-worker retry loop map directly onto the driver's concurrency, exclusion_key and backoff, so all of that goes; the node_addr guard that keeps two tasks off the same host survives as the exclusion key. The reboot-aware branch, which used to infer "this attempt should not count" by comparing task.retry before and after and rolling it back, becomes a plain TaskDefer. An add_node that raises is now treated the same as one that reports failure — it consumed no retry before, so a permanently crashing add could never reach its ceiling. --- .../services/tasks_runner_node_add.py | 221 +++++------------- tests/unit/tasks/test_max_retry_semantics.py | 71 +++--- tests/unit/tasks/test_retry_ceiling.py | 13 +- 3 files changed, 94 insertions(+), 211 deletions(-) diff --git a/simplyblock_core/services/tasks_runner_node_add.py b/simplyblock_core/services/tasks_runner_node_add.py index 9756988651..83210f2e95 100644 --- a/simplyblock_core/services/tasks_runner_node_add.py +++ b/simplyblock_core/services/tasks_runner_node_add.py @@ -1,14 +1,17 @@ # coding=utf-8 import socket -import threading import time -from concurrent.futures import ThreadPoolExecutor from simplyblock_core import db_controller, storage_node_ops, utils, constants -from simplyblock_core.controllers import tasks_controller from simplyblock_core.models.job_schedule import JobSchedule from simplyblock_core.models.cluster import Cluster +from simplyblock_core.services.task_runner_base import ( + RunnerSpec, + TaskDefer, + TaskRetry, + serve, +) logger = utils.get_logger(__name__) @@ -24,67 +27,6 @@ # cluster-create / expansion fan-out can't exhaust the runner host. MAX_CONCURRENT_NODE_ADDS = constants.NODE_ADD_MAX_PARALLEL -# uuids of node-add tasks a worker on THIS host is currently driving, so the -# dispatch loop never hands the same task to two workers. (Cross-host -# duplicate execution is separately prevented by the per-task lease in -# tasks_controller.claim_task.) -_inflight = set() -_inflight_lock = threading.Lock() - -# target node_addr values currently being driven by a worker, guarded by the -# same lock. The concurrency model above ("different tasks target different -# nodes, no shared state") breaks if TWO task records ever target the SAME -# host — e.g. a caller's retried HTTP request creating a second FN_NODE_ADD -# task before tasks_controller._validate_new_task_node_add existed to block -# it, or any task created before that dedup shipped. Without this, both -# tasks' add_node() calls race the same host's config-slot classify-then- -# create logic milliseconds apart (2026-07-23: two threads, 4ms apart, -# produced 6 node records for a 4-slot host). This is belt-and-suspenders -# for the task-creation-time dedup — it protects against any duplicate task -# that already exists, regardless of how it got created. -_inflight_addrs = set() - - -def process_task(task, cl): - if task.canceled: - task.function_result = "canceled" - task.status = JobSchedule.STATUS_DONE - task.write_to_db(db.kv_store) - return False - - if 0 <= task.max_retry <= task.retry: - task.function_result = "max retry reached" - task.status = JobSchedule.STATUS_DONE - task.write_to_db(db.kv_store) - return True - - if db.get_cluster_by_id(cl.get_id()).status == Cluster.STATUS_IN_ACTIVATION: - task.function_result = "Cluster is in_activation, waiting" - task.status = JobSchedule.STATUS_NEW - task.write_to_db(db.kv_store) - return False - - if task.status != JobSchedule.STATUS_RUNNING: - task.status = JobSchedule.STATUS_RUNNING - task.write_to_db(db.kv_store) - - try: - res = storage_node_ops.add_node(**task.function_params) - msg = f"Node add result: {res}" - logger.info(msg) - task.function_result = msg - if res: - task.status = JobSchedule.STATUS_DONE - else: - task.retry += 1 - task.status = JobSchedule.STATUS_SUSPENDED - task.write_to_db(db.kv_store) - return True - except Exception as e: - logger.error(e) - return False - - # Applying the CPU topology during add_node makes the node reboot # (kubeletconfig / MCP update). The in-flight attempt then fails, but the # right reaction is neither a quick blind retry (the node is down for @@ -96,11 +38,23 @@ def process_task(task, cl): NODE_REBOOT_POLL_SEC = 15 +def _node_addr(task): + """The target host of an add. Two task records must never drive the same + one concurrently: the concurrency model above ("different tasks target + different nodes, no shared state") breaks if they do — two add_node() calls + race the same host's config-slot classify-then-create logic milliseconds + apart (2026-07-23: two threads, 4ms apart, produced 6 node records for a + 4-slot host). tasks_controller._validate_new_task_node_add should stop such + a task from being created at all; this is the backstop for any that already + exists, however it got there.""" + return (task.function_params or {}).get("node_addr") + + def _node_api_reachable(task, timeout=5): """TCP-level reachability of the node agent (host:port from the task's node_addr). During add the StorageNode record may not exist yet, so this intentionally checks the address, not the DB object.""" - addr = (task.function_params or {}).get("node_addr", "") + addr = _node_addr(task) or "" if ":" not in addr: return True # can't tell — let the normal retry path decide host, _, port = addr.rpartition(":") @@ -111,7 +65,7 @@ def _node_api_reachable(task, timeout=5): return False -def _wait_node_reachable(task, task_uuid): +def _wait_node_reachable(task): """After a failed attempt, if the node is unreachable (rebooting for the CPU-topology change), wait for it to answer again — up to NODE_REBOOT_WAIT_MAX_SEC — instead of consuming retries against a node @@ -119,121 +73,56 @@ def _wait_node_reachable(task, task_uuid): if _node_api_reachable(task): return False logger.info( - f"Node-add task {task_uuid}: node agent unreachable (rebooting for " + f"Node-add task {task.uuid}: node agent unreachable (rebooting for " f"CPU topology?); waiting up to {NODE_REBOOT_WAIT_MAX_SEC}s for it to return") deadline = time.time() + NODE_REBOOT_WAIT_MAX_SEC while time.time() < deadline: time.sleep(NODE_REBOOT_POLL_SEC) if _node_api_reachable(task): - logger.info(f"Node-add task {task_uuid}: node agent reachable again; retrying add") + logger.info(f"Node-add task {task.uuid}: node agent reachable again; retrying add") return True logger.warning( - f"Node-add task {task_uuid}: node agent still unreachable after " + f"Node-add task {task.uuid}: node agent still unreachable after " f"{NODE_REBOOT_WAIT_MAX_SEC}s; resuming normal retry schedule") return True -def _run_task(task_uuid, cluster_id, node_addr): - """Worker thread: drive one node-add task to completion (or suspension), - then drop it from the in-flight set so a later cycle can retry it. - - Guarded against BaseException — add_node -> write_to_db calls exit(1) on a - DB write failure, which surfaces here as SystemExit; it must be logged and - contained to this worker, never allowed to kill the service loop or leave - the task stuck in the in-flight set. - """ - delay_seconds = constants.TASK_EXEC_INTERVAL_SEC +def process_task(task): try: - while True: - # Re-fetch for fresh FDB state (the task may have been canceled). - task = db.get_task_by_id(task_uuid) - if task is None or task.status == JobSchedule.STATUS_DONE: - break - cl = db.get_cluster_by_id(cluster_id) - # Lease gate: skip a task another live runner host owns. - if not tasks_controller.claim_task(task): - logger.info(f"Node-add task {task_uuid} owned by another runner host; skipping") - break - # add_node blocks for many minutes with no task writes; heartbeat - # the lease so another runner host never sees it stale mid-add. - retry_before = task.retry - with tasks_controller.task_lease_heartbeat(task): - res = process_task(task, cl) - if res: - if task.status == JobSchedule.STATUS_DONE: - break - # Reboot-aware handling: an attempt that failed because the node - # went down (CPU-topology reboot) is expected, not a real failure. - # add_node catches the interrupted spdk_process_start and RETURNS - # False, so process_task already consumed a retry — roll it back so - # the one guaranteed topology reboot per node doesn't eat the - # retry budget. Then wait for the agent to answer and retry - # promptly on a fresh schedule (no blind fast-retry, no runaway - # backoff). The re-run is idempotent: add_node cleans up its own - # stale IN_CREATION record on re-entry. - if _wait_node_reachable(task, task_uuid): - if task.retry > retry_before: - task.retry = retry_before - if task.status != JobSchedule.STATUS_DONE: - task.status = JobSchedule.STATUS_SUSPENDED - task.write_to_db(db.kv_store) - delay_seconds = constants.TASK_EXEC_INTERVAL_SEC - continue - if not res: - # Cap the exponential backoff so a permanently failing node-add - # can't grow the sleep without bound. - delay_seconds = min( - delay_seconds * 2, - constants.RESTART_TASK_EXEC_INTERVAL_MAX_SEC, - ) - time.sleep(delay_seconds) - except BaseException as e: - logger.error(f"Node-add task {task_uuid} processing crashed: {e}") - logger.exception(e) - finally: - with _inflight_lock: - _inflight.discard(task_uuid) - if node_addr: - _inflight_addrs.discard(node_addr) + res = storage_node_ops.add_node(**task.function_params) + msg = f"Node add result: {res}" + logger.info(msg) + except Exception as e: + logger.error(e) + res, msg = False, f"Node add raised: {e}" + + if res: + task.function_result = msg + return + + # The one guaranteed topology reboot per node must not eat the retry + # budget: add_node catches the interrupted spdk_process_start and reports + # failure, so wait for the agent to answer and re-attempt on a fresh + # schedule instead. The re-run is idempotent — add_node cleans up its own + # stale IN_CREATION record on re-entry. + if _wait_node_reachable(task): + raise TaskDefer(f"{msg} (node agent was rebooting)") + + raise TaskRetry(msg) + + +SPEC = RunnerSpec( + name="tasks-runner-node-add", + function_names=[JobSchedule.FN_NODE_ADD], + handler=process_task, + is_eligible=lambda task, cluster: cluster.status != Cluster.STATUS_IN_ACTIVATION, + concurrency=MAX_CONCURRENT_NODE_ADDS, + exclusion_key=_node_addr, +) def main(): - logger.info("Starting Tasks runner node add...") - - executor = ThreadPoolExecutor(max_workers=MAX_CONCURRENT_NODE_ADDS) - - while True: - clusters = db.get_clusters() - if not clusters: - logger.error("No clusters found!") - else: - for cl in clusters: - tasks = db.get_job_tasks(cl.get_id(), reverse=False) - for task in tasks: - if task.function_name != JobSchedule.FN_NODE_ADD: - continue - if task.status == JobSchedule.STATUS_DONE: - continue - node_addr = (task.function_params or {}).get("node_addr") - # Dispatch to a worker once; skip if a worker on this host is - # already driving it. Excess tasks queue in the executor and - # run as workers free up. Also skip if a DIFFERENT task - # already targets the SAME node_addr: tasks_controller's - # creation-time dedup should prevent that task from ever - # existing, but this is the backstop — two tasks racing the - # same host's config-slot classify-then-create logic - # produced duplicate storage-node records (2026-07-23). - with _inflight_lock: - if task.uuid in _inflight: - continue - if node_addr and node_addr in _inflight_addrs: - continue - _inflight.add(task.uuid) - if node_addr: - _inflight_addrs.add(node_addr) - executor.submit(_run_task, task.uuid, cl.get_id(), node_addr) - - time.sleep(constants.TASK_EXEC_INTERVAL_SEC) + serve(SPEC) if __name__ == "__main__": diff --git a/tests/unit/tasks/test_max_retry_semantics.py b/tests/unit/tasks/test_max_retry_semantics.py index 801f7711be..761e8b21e9 100644 --- a/tests/unit/tasks/test_max_retry_semantics.py +++ b/tests/unit/tasks/test_max_retry_semantics.py @@ -10,9 +10,10 @@ The invariant under test is the basic task-scheduler semantic: a task whose ``retry`` has reached its ceiling must *terminate* (``STATUS_DONE``) instead of looping forever, and it must not perform its side-effecting work on that final -poll. Below the ceiling the task still advances. See -:mod:`test_retry_ceiling` for the backup runner's version of the same guard and -the static cross-runner check. +poll. Below the ceiling the task still advances. See :mod:`test_retry_ceiling` +for the cross-runner discovery check, and +:mod:`tests.unit.tasks.test_task_runner_base` for the ceiling as enforced by the +shared driver, which the migrated runners delegate it to. """ from unittest.mock import MagicMock @@ -23,6 +24,7 @@ from simplyblock_core.models.storage_node import StorageNode import simplyblock_core.services.tasks_runner_node_add as node_add_runner +from simplyblock_core.services.task_runner_base import TaskDefer, TaskRetry import simplyblock_core.services.tasks_runner_restart as restart_runner @@ -49,54 +51,57 @@ def _task(function_name, retry, max_retry, **params): # -------------------------------------------------------------------------- # tasks_runner_node_add.process_task +# +# The ceiling itself now lives in the shared driver (see +# tests/unit/tasks/test_task_runner_base.py); what this runner still decides is +# whether a failed add counts against it at all. # -------------------------------------------------------------------------- -def test_node_add_max_retry_finishes_without_adding(monkeypatch): - """retry >= max_retry must finish the task and never call add_node.""" +def test_node_add_success_completes(monkeypatch): sops = MagicMock() + sops.add_node.return_value = True monkeypatch.setattr(node_add_runner, "storage_node_ops", sops) - monkeypatch.setattr(node_add_runner, "db", MagicMock()) - - task = _task(JobSchedule.FN_NODE_ADD, retry=3, max_retry=3, node_id="node-1") - res = node_add_runner.process_task(task, MagicMock()) - assert res is True - assert task.status == JobSchedule.STATUS_DONE - assert "max retry" in task.function_result - sops.add_node.assert_not_called() + task = _task(JobSchedule.FN_NODE_ADD, retry=0, max_retry=3, node_id="node-1") + assert node_add_runner.process_task(task) is None + sops.add_node.assert_called_once_with(node_id="node-1") -def test_node_add_below_ceiling_dispatches(monkeypatch): - """Below the ceiling the task still runs its step (add_node).""" +def test_node_add_failure_counts_a_retry(monkeypatch): + """A failed add against a responsive node advances retry, so the driver's + ceiling can eventually bind.""" sops = MagicMock() - sops.add_node.return_value = True + sops.add_node.return_value = False monkeypatch.setattr(node_add_runner, "storage_node_ops", sops) - # get_cluster_by_id().status is a Mock != STATUS_IN_ACTIVATION, so the - # in-activation gate is not taken and the task proceeds to add_node. - monkeypatch.setattr(node_add_runner, "db", MagicMock()) - - task = _task(JobSchedule.FN_NODE_ADD, retry=0, max_retry=3, node_id="node-1") - res = node_add_runner.process_task(task, MagicMock()) + monkeypatch.setattr(node_add_runner, "_wait_node_reachable", lambda task: False) - assert res is True - sops.add_node.assert_called_once_with(node_id="node-1") - assert task.status == JobSchedule.STATUS_DONE + task = _task(JobSchedule.FN_NODE_ADD, retry=1, max_retry=3, node_id="node-1") + with pytest.raises(TaskRetry): + node_add_runner.process_task(task) -def test_node_add_failure_below_ceiling_suspends_and_counts_retry(monkeypatch): - """A failed add below the ceiling suspends and advances retry (so the - ceiling can eventually bind), rather than finishing the task.""" +def test_node_add_failure_during_a_reboot_does_not_count(monkeypatch): + """The CPU-topology reboot is expected, not a failure: waiting it out must + not burn one of the task's retries.""" sops = MagicMock() sops.add_node.return_value = False monkeypatch.setattr(node_add_runner, "storage_node_ops", sops) - monkeypatch.setattr(node_add_runner, "db", MagicMock()) + monkeypatch.setattr(node_add_runner, "_wait_node_reachable", lambda task: True) task = _task(JobSchedule.FN_NODE_ADD, retry=1, max_retry=3, node_id="node-1") - res = node_add_runner.process_task(task, MagicMock()) + with pytest.raises(TaskDefer): + node_add_runner.process_task(task) - assert res is True # processed; the loop keeps polling - assert task.status == JobSchedule.STATUS_SUSPENDED - assert task.retry == 2 + +def test_node_add_exception_is_handled_like_a_failed_add(monkeypatch): + sops = MagicMock() + sops.add_node.side_effect = RuntimeError("boom") + monkeypatch.setattr(node_add_runner, "storage_node_ops", sops) + monkeypatch.setattr(node_add_runner, "_wait_node_reachable", lambda task: False) + + task = _task(JobSchedule.FN_NODE_ADD, retry=0, max_retry=3, node_id="node-1") + with pytest.raises(TaskRetry, match="boom"): + node_add_runner.process_task(task) # -------------------------------------------------------------------------- diff --git a/tests/unit/tasks/test_retry_ceiling.py b/tests/unit/tasks/test_retry_ceiling.py index 8d2fa00a3f..50f0206df4 100644 --- a/tests/unit/tasks/test_retry_ceiling.py +++ b/tests/unit/tasks/test_retry_ceiling.py @@ -192,17 +192,6 @@ def _wire_base(runner, monkeypatch, task): # tree, so a new retry-driven runner shows up as a failing case until a spec is # added here. -def _spec_node_add(runner, monkeypatch): - task = _make_task(JobSchedule.FN_NODE_ADD) - _wire_base(runner, monkeypatch, task) - monkeypatch.setattr(runner, "ThreadPoolExecutor", _InlineExecutor) - monkeypatch.setattr(runner, "_inflight", set()) - # add_node fails (returns falsy) every cycle. - monkeypatch.setattr(runner.storage_node_ops, "add_node", - MagicMock(return_value=False)) - return task - - def _spec_restart(runner, monkeypatch): task = _make_task(JobSchedule.FN_NODE_RESTART) _db, _cluster, node = _wire_base(runner, monkeypatch, task) @@ -278,7 +267,6 @@ def _get_node(node_id): # name -> spec for the runners driven through their real main() loop. _MAIN_DRIVEN_SPECS = { - "tasks_runner_node_add.py": _spec_node_add, "tasks_runner_restart.py": _spec_restart, "tasks_runner_batch_migration.py": _spec_batch_migration, } @@ -303,6 +291,7 @@ def _get_node(node_id): "tasks_runner_sync_lvol_del.py", "tasks_runner_backup.py", "tasks_runner_cluster_expand.py", + "tasks_runner_node_add.py", } # Runners that increment task.retry but are intentionally UNBOUNDED: the From af20738235407d9b8dda23f1330b83e96d1a5a03 Mon Sep 17 00:00:00 2001 From: Max Schettler Date: Wed, 5 Aug 2026 13:41:16 +0200 Subject: [PATCH 14/25] Update handover document --- HANDOFF.md | 133 ++++++++++++++++++++++++++++------------------------- 1 file changed, 70 insertions(+), 63 deletions(-) diff --git a/HANDOFF.md b/HANDOFF.md index b56d0bf649..30592ef0b5 100644 --- a/HANDOFF.md +++ b/HANDOFF.md @@ -1,80 +1,87 @@ # Task-runner rework — handoff -Continuation notes for picking up the `rework-task-runners` branch on another -machine. The full design lives in **`TASK_RUNNER_REWORK_PLAN.md`** (same dir) — -read it first; this file is just the current state + how to continue. +Continuation notes for the `rework-task-runners` branch. The full design lives +in **`TASK_RUNNER_REWORK_PLAN.md`** (same dir) — read it first; this file is +just the current state + how to continue. ## Where the work is -Branch `rework-task-runners`, rebased onto `origin/main`. Five commits: +Branch `rework-task-runners`, rebased onto `origin/main` (2026-08-05, commit +`4cb16a20b`). Pre-rebase safety backup: branch `backup/rework-pre-rebase3`. ``` -6ea41859c Introduce generic task runner (B0: driver + lease heartbeat + tests) -8458a5e44 Reclassify backup policy evaluation… (A5: backup_merge -> backup_merge_service) -6079ed010 Apply task lease consistently… (A3: claim_task on the lease-less runners) -4882cecac Unify task retry semantics (A2: canonical 0 <= max_retry <= retry) -9da6cd5f7 Consistently crash task runners on DB… (A1: remove DB-error masking) +fa26b7121 Migrate node add runner… (B3) +f954be2b9 Migrate backup and cluster expand… (B2) +d6cc94478 Migrate lvol sync runner… (B1d) +b436fa79d Migrate replication cutover runner… (B1c) +9c9118761 Migrate JC compression resume runner… (B1b) +d17c20064 Migrate FDB backup runner… (B1a) +3899c2452 Reset task result before each handler attempt +ad4768462 Introduce generic task runner (B0) + ``` -Pre-rebase safety backup: branch `backup/rework-pre-rebase2` (`61cd76770`). - -## Status: Phase A done, B0 done, Phase B (migrations) NOT started - -- **A1/A2/A3/A5** — done. **A4** was folded into Phase B (no commit — see plan). -- **B0** — `simplyblock_core/services/task_runner_base.py` (the `TaskRunner` - driver: loop, single post-claim re-fetch, eligibility + lease pre-run gates, - `task_lease_heartbeat` around the handler, retry ceiling + backoff, serial / - opt-in concurrency) + `tests/unit/tasks/test_task_runner_base.py`. **Done.** -- **B1…B9** — migrate each runner onto the driver, one commit each, in the order - listed in the plan (`fdb_backup, jc_comp, replication_final, sync_lvol_del` → - `backup, cluster_expand` → `node_add` → migration trio → `port_allow` → - `restart` → `lvol_migration` → `node_removal` → `batch_migration`). **Not started.** - -## Gotchas (read before continuing) - -- **`tests/unit/tasks/test_retry_ceiling.py` HANGS — and it hangs on `origin/main` - itself.** Upstream restructured the runners it drives (module-level loops, - `task_runner`→`process_task`) without updating it. Do NOT run the full - `tests/unit/tasks/` dir until it is skipped/removed. It exercises the old - `main()` loops that Phase B deletes; plan is to retire it as runners migrate. -- **Upstream already reworked the runners**: renamed `task_runner`→`process_task` - and added `tasks_controller.task_lease_heartbeat` to the six long-blocking - runners. B0's driver already folds heartbeat in, so migrating a runner should - DELETE its per-runner `process_task` + `claim_task` + `task_lease_heartbeat` - boilerplate in favor of `serve(SPEC)`. -- **Two new upstream runners** to migrate: `tasks_runner_node_removal.py` - (`FN_NODE_REMOVAL`) and `tasks_runner_batch_migration.py` (`FN_LVOL_BATCH_MIG`). -- Each migrated `main()` collapses to `serve(SPEC)`; each handler becomes **void** - and signals via `TaskDefer` / `TaskRetry` / `TaskAbort` (see plan §B0). Keep the - handler importable — tests call it directly. - -## Environment setup on the new machine - -```bash -git fetch -git checkout rework-task-runners -pip install -e . # editable install -# FoundationDB 7.3.3 client library (libfdb_c) must be installed on the host -``` - -## Verification (was NOT runnable on the origin machine — offline) - -The origin box had no network and a wiped `.tox`, so lint/types/tox could not run -there; the driver was verified with an ad-hoc stub harness only. **Re-run the real -checks on the new machine:** +## Status: Phase A done, B0 done, B1–B3 done + +Migrated onto the driver (7): `fdb_backup`, `jc_comp`, `replication_final`, +`sync_lvol_del`, `backup`, `cluster_expand`, `node_add`. + +**Remaining (B4…B9), in order:** + +4. migration trio — `migration`, `new_dev_migration`, `failed_migration` + (serial; `get_active_node_mig_task` / same-node-sibling gating becomes + `is_eligible`) +5. `port_allow` (large; its `is_eligible` omits the `IN_ACTIVATION` check — the + documented opt-out — and keeps the recovery logic) +6. `restart` (concurrency + per-node `exclusion_key`; `is_auto_restart_paused` + and `fd_dead_recovery_allowed` become part of `is_eligible`) +7. `lvol_migration` (largest; migrate the loop/lease/retry shell only, leave the + snapshot-copy state machine intact) +8. `node_removal` (serial, already leased, unbounded) +9. `batch_migration` (migrate the loop/lease shell, leave group orchestration) + +## What the driver grew during B1–B3 + +Beyond what the plan describes, `task_runner_base` gained two things the +migrations needed: + +- **`function_result` is cleared before each handler attempt**, so a task that + fails and later succeeds doesn't finish carrying the stale failure message. +- **`RunnerSpec.on_finish(task)`** — cleanup called after the task reaches + STATUS_DONE and is written, on *every* terminal path (success, `TaskAbort`, + cancel, retry ceiling). Needed because a handler never sees the terminal + paths the driver owns, yet resources it holds must still be released: + `sync_lvol_del` frees the primary's del-sync lock there, `backup` fails/ + un-merges the backup resource there. Both are written to be no-ops when the + handler already finished the resource. + +## Gotchas + +- **`tests/unit/tasks/test_retry_ceiling.py` no longer hangs** — upstream fixed + it before this rebase. It parametrizes over runners *discovered from source* + by the presence of `.retry += 1`, so a runner migrating to the driver silently + drops out of it. Each migration therefore also moves the runner into that + file's `_DRIVER_MIGRATED` set, which is asserted to really have handed the + retry counter over (`test_migrated_runners_delegate_retry`). +- Per-runner behaviour tests for migrated runners live in + **`tests/unit/tasks/test_runner_specs.py`** (one section per runner: handler + outcome vocabulary + eligibility + `on_finish`). Extend it as you migrate. +- Several migrations fix latent bugs (a failure path that suspended without ever + incrementing retry, so a declared `max_retry` could never bind). Each is called + out in its commit message — keep doing that rather than folding them in + silently. + +## Verification ```bash tox run-parallel -e lint,types -tox run -e unit -- tests/unit/tasks/test_task_runner_base.py \ - tests/unit/tasks/test_max_retry_semantics.py \ - tests/unit/test_task_lease.py \ - tests/unit/test_lvol_sync_op_task.py -# NOTE: do not add tests/unit/tasks/test_retry_ceiling.py — it hangs (see gotchas) +tox run -e unit -- tests/unit/tasks/ tests/unit/test_lvol_sync_op_task.py +tox run -e unit # full unit tier, ~45s, currently green ``` -Follow the `tox-verify` skill / `AGENTS.md` for the full workflow. +Do not run `tox run` (the integration tier is broken independently of this work). ## Before opening the PR -Delete these two handoff files (`HANDOFF.md`, `TASK_RUNNER_REWORK_PLAN.md`) — they -are transfer artifacts, not part of the change. +Delete these two handoff files (`HANDOFF.md`, `TASK_RUNNER_REWORK_PLAN.md`) — +they are transfer artifacts, not part of the change. From 5176ecba2b14b776e1cad0ea141b15647a1e8b3e Mon Sep 17 00:00:00 2001 From: Max Schettler Date: Wed, 5 Aug 2026 14:19:22 +0200 Subject: [PATCH 15/25] Commit task transitions by compare-and-set MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The driver wrote task state with task.write_to_db() on the copy it fetched before running the handler. A handler runs for minutes, and the row changes underneath it: set_node_status(ONLINE) cancels restart tasks via cancel_pending_node_restart_tasks, an operator cancels a task, another host's heartbeat stamps the lease. Writing the whole stale object back reverted all of it — an un-canceled task and a reclaimed lease are the two lost updates behind the 2026-07-29 double restart, and the reason upstream converted the restart runner's own writes to CAS. Every transition now runs as a mutator against the current row, refusing a row another actor has finished (and, for non-terminal transitions, one it has canceled), and reporting whether it won. Retry increments land on the fresh count instead of a stale one. on_finish only runs for the caller that won the terminal transition, so a resource is not released twice. Dispatch gets the same treatment: serialized execution now submits to the pool and waits rather than running inline. The inline path registered nothing in the inflight map, which is exactly the split that let a dispatch-mode flip re-enter a task still running and force-shut an already-recovered node. Serialization becomes a per-task predicate, since node restart chooses its mode from live cluster state rather than a fixed concurrency. Tests assert on the committed row via a store that models both write paths, so each of these reproduces as a failure against the previous implementation. --- simplyblock_core/services/task_runner_base.py | 151 ++++++-- tests/unit/tasks/test_task_runner_base.py | 356 ++++++++++++++---- 2 files changed, 417 insertions(+), 90 deletions(-) diff --git a/simplyblock_core/services/task_runner_base.py b/simplyblock_core/services/task_runner_base.py index 21cbd929c4..7d7335906f 100644 --- a/simplyblock_core/services/task_runner_base.py +++ b/simplyblock_core/services/task_runner_base.py @@ -26,10 +26,25 @@ - **raise** :class:`TaskAbort` — a permanent, non-retryable stop (missing param, object gone, "not needed"). Finish the task (``STATUS_DONE``) with the reason. +Two task fields ARE the handler's to set: ``function_result`` (the message the +outcome is recorded with) and ``function_params`` (where a multi-cycle handler +records progress — ``recovery_started``, ``merge_started``, ``fail_count``). +Both are carried onto the row when the driver commits the outcome. + +A handler that does something destructive should re-read the task immediately +before doing it. The driver's pre-run re-fetch is authoritative for the +*lifecycle* decisions it makes, but it happens before the handler starts, and by +the time a long handler reaches its point of no return the task may have been +canceled. + DB errors are deliberately NOT caught: an unhandled ``get_clusters`` / ``get_job_tasks`` failure propagates out of :func:`serve`, exits the process non-zero, and lets the orchestrator restart it with a fresh FDB connection. + +Task writes are compare-and-set, never full-object writes: see +:meth:`TaskRunner._cas`. """ +import datetime import threading import time from concurrent.futures import ThreadPoolExecutor @@ -91,8 +106,15 @@ class RunnerSpec: # Optional cleanup, called once the task has reached STATUS_DONE and been # written — whichever way it got there (handler success, TaskAbort, cancel # or retry ceiling). For releasing state the task held, which would - # otherwise leak on the terminal paths the handler never sees. + # otherwise leak on the terminal paths the handler never sees. Runs only + # for the caller that won the terminal transition. on_finish: Optional[Callable[[JobSchedule], None]] = None + # Optional per-task, per-cycle "must this one run to completion before the + # loop moves on?". Defaults to serializing exactly when the pool has a + # single worker. A runner whose mode depends on live cluster state (node + # restart fans out only for a drained suspension or a fully-dead failure + # domain) supplies a predicate instead of a fixed concurrency. + serialize: Optional[Callable[[JobSchedule, Any], bool]] = None def __post_init__(self) -> None: if self.concurrency < 1: @@ -105,10 +127,8 @@ class TaskRunner: def __init__(self, spec: RunnerSpec): self.spec = spec - self._executor = ( - ThreadPoolExecutor(max_workers=spec.concurrency, - thread_name_prefix=spec.name) - if spec.concurrency > 1 else None) + self._executor = ThreadPoolExecutor(max_workers=spec.concurrency, + thread_name_prefix=spec.name) self._lock = threading.Lock() # task uuid -> Future-in-flight guard (this host), so the dispatch loop # never hands the same task to two workers. Cross-host duplicate @@ -147,10 +167,6 @@ def _dispatch(self, task: JobSchedule, cluster: Any) -> None: if time.time() < self._next_attempt.get(uuid, 0): return - if self._executor is None: - self._process(task, cluster) - return - with self._lock: if uuid in self._inflight: return @@ -160,7 +176,21 @@ def _dispatch(self, task: JobSchedule, cluster: Any) -> None: self._inflight.add(uuid) if key is not None: self._inflight_keys[key] = uuid - self._executor.submit(self._process_worker, task, cluster) + + # Single dispatch path: serialized execution submits to the pool and + # waits, rather than running inline. A split — one branch registering + # in-flight and another not — is what let a dispatch-mode flip + # mid-restart re-enter a task that was still running, and force-shut an + # already-recovered node (2026-07-29 double restart). Going through the + # registry either way makes a flip harmless in both directions. + future = self._executor.submit(self._process_worker, task, cluster) + if self._serialized(task, cluster): + future.result() + + def _serialized(self, task: JobSchedule, cluster: Any) -> bool: + if self.spec.serialize is not None: + return self.spec.serialize(task, cluster) + return self.spec.concurrency == 1 def _process_worker(self, task: JobSchedule, cluster: Any) -> None: # A worker crash must be contained to this task, never kill the service @@ -187,10 +217,10 @@ def _process(self, task: JobSchedule, cluster: Any) -> None: logger.info(f"{self.spec.name}: task {uuid} owned by another runner host; skipping") return - # Authoritative re-fetch AFTER the claim: claim_task mutated the DB copy - # (owner / updated_at) but not this local object, and write_to_db writes - # the whole object — reading the fresh, lease-stamped copy here is what - # keeps a later write from clobbering the owner. + # Authoritative re-fetch AFTER the claim: claim_task mutated the DB row + # (owner / updated_at) but not this local object, and the lifecycle + # decisions below — canceled, retry ceiling — must be made on the row as + # it stands, not on whatever the dispatch loop happened to read. task = db.get_task_by_id(uuid) if task is None or task.status == JobSchedule.STATUS_DONE: self._forget(uuid) @@ -204,8 +234,11 @@ def _process(self, task: JobSchedule, cluster: Any) -> None: return if task.status != JobSchedule.STATUS_RUNNING: - task.status = JobSchedule.STATUS_RUNNING - task.write_to_db(db.kv_store) + running = self._cas(task, self._to(JobSchedule.STATUS_RUNNING)) + if running is None: + self._forget(uuid) + return + task = running # Drop the previous attempt's result so a task that fails and later # succeeds does not finish carrying the stale failure message. Handlers @@ -234,8 +267,62 @@ def _process(self, task: JobSchedule, cluster: Any) -> None: # -- outcome transitions (the only places task state is mutated) -------- + def _cas(self, task: JobSchedule, apply: Callable[[JobSchedule], None], + terminal: bool = False) -> Optional[JobSchedule]: + """Commit a lifecycle transition onto the task row as it exists NOW. + + Never a full-object ``write_to_db`` of ``task``: the driver fetched that + copy before the handler ran, and a handler runs for minutes (node add, + restart, migration). Writing it back reinstates every field another + actor changed meanwhile — it un-cancels a task that + ``cancel_pending_node_restart_tasks`` canceled when the node came back + ONLINE, and reclaims a lease another host has since taken. That pair of + lost updates is what re-ran a restart against an already-recovered node + (2026-07-29 double restart). + + Only the two fields a handler owns are carried over from ``task``: + ``function_result`` and ``function_params`` (where handlers record + progress like ``recovery_started`` / ``merge_started``, which must + survive so the next attempt does not re-issue the RPC). + + Returns the committed task, or None if another actor already owns the + outcome — the caller must stop driving it. + """ + result = task.function_result + params = task.function_params + now = str(datetime.datetime.now(datetime.timezone.utc)) + won = {"ok": False} + + def _mutate(fresh: JobSchedule): + if fresh.status == JobSchedule.STATUS_DONE: + return False + # A terminal transition is allowed to finish a canceled task — + # that IS the cancellation being carried out. A non-terminal one + # would be reviving it. + if not terminal and fresh.canceled: + return False + fresh.function_result = result + fresh.function_params = params + apply(fresh) + fresh.updated_at = now + won["ok"] = True + return True + + committed = db.atomic_update(task, _mutate) + if committed is None or not won["ok"]: + logger.info( + f"{self.spec.name}: task {task.uuid} was finished or canceled " + f"concurrently; another actor owns the outcome") + return None + return committed + + @staticmethod + def _to(status: str) -> Callable[[JobSchedule], None]: + def _apply(task: JobSchedule) -> None: + task.status = status + return _apply + def _succeed(self, task: JobSchedule) -> None: - task.status = JobSchedule.STATUS_DONE if not task.function_result: task.function_result = "completed" self._write_terminal(task) @@ -244,19 +331,21 @@ def _finish(self, task: JobSchedule, result: str) -> None: """Terminal DONE for a non-handler-success reason (canceled, max retry, abort).""" task.function_result = result - task.status = JobSchedule.STATUS_DONE self._write_terminal(task) def _write_terminal(self, task: JobSchedule) -> None: - task.write_to_db(db.kv_store) + committed = self._cas(task, self._to(JobSchedule.STATUS_DONE), terminal=True) self._forget(task.uuid) - if self.spec.on_finish is None: + if committed is None or self.spec.on_finish is None: + # Losing the transition means someone else finished the task and + # owns its cleanup too; running it here would release the resource + # twice. return # Cleanup runs after the terminal write, so a hook that inspects the # task's own state (a lock held until no active task remains) sees it # as finished. A failing hook must not take the loop down with it. try: - self.spec.on_finish(task) + self.spec.on_finish(committed) except Exception as e: # noqa: BLE001 - cleanup failure is not fatal logger.error(f"{self.spec.name}: task {task.uuid} on_finish failed: {e}") logger.exception(e) @@ -264,17 +353,25 @@ def _write_terminal(self, task: JobSchedule) -> None: def _defer(self, task: JobSchedule, reason: str) -> None: if reason: task.function_result = reason - task.status = JobSchedule.STATUS_SUSPENDED - task.write_to_db(db.kv_store) + if self._cas(task, self._to(JobSchedule.STATUS_SUSPENDED)) is None: + self._forget(task.uuid) + return self._clear_backoff(task.uuid) def _fail(self, task: JobSchedule, reason: str) -> None: - task.retry += 1 task.function_result = reason - task.status = JobSchedule.STATUS_SUSPENDED - task.write_to_db(db.kv_store) + + def _apply(fresh: JobSchedule) -> None: + fresh.retry += 1 + fresh.status = JobSchedule.STATUS_SUSPENDED + + committed = self._cas(task, _apply) + if committed is None: + self._forget(task.uuid) + return + # Back off on the committed retry count, not the stale local one. with self._lock: - self._next_attempt[task.uuid] = time.time() + self._backoff_delay(task.retry) + self._next_attempt[task.uuid] = time.time() + self._backoff_delay(committed.retry) # -- bookkeeping -------------------------------------------------------- diff --git a/tests/unit/tasks/test_task_runner_base.py b/tests/unit/tasks/test_task_runner_base.py index bc3a2509bc..bc0c22bc5d 100644 --- a/tests/unit/tasks/test_task_runner_base.py +++ b/tests/unit/tasks/test_task_runner_base.py @@ -2,12 +2,20 @@ """Unit tests for the shared task-runner driver (``task_runner_base``). These exercise the per-task lifecycle (``_process``) and the dispatch loop -(``run``) directly, with ``db`` and ``claim_task`` mocked. They pin the handler -contract: return → DONE, TaskDefer → suspend without a retry, TaskRetry / -unexpected Exception → suspend + retry + backoff, TaskAbort → DONE; the pre-run -skip-gates (eligibility, lease); and that a handler exception never escapes the -loop while a DB error does. +(``run``) directly, against a fake store that models FoundationDB's two write +paths faithfully. They pin the handler contract: return → DONE, TaskDefer → +suspend without a retry, TaskRetry / unexpected Exception → suspend + retry + +backoff, TaskAbort → DONE; the pre-run skip-gates (eligibility, lease); that a +handler exception never escapes the loop while a DB error does; and — the part +that needs a real store to observe — that no transition can revert what another +actor committed while the handler was running. + +Assertions read the committed row (``store.row()``), not the caller's copy: the +driver's writes are compare-and-set against the current row and deliberately do +not mutate the stale object it was holding. """ +import copy +import threading import time from unittest.mock import MagicMock @@ -17,13 +25,6 @@ import simplyblock_core.services.task_runner_base as trb -@pytest.fixture(autouse=True) -def _no_task_writes(monkeypatch): - """Task writes would dereference a None kv_store; the driver's state - transitions are asserted on the in-memory task object instead.""" - monkeypatch.setattr(JobSchedule, "write_to_db", MagicMock()) - - def _task(status=JobSchedule.STATUS_NEW, retry=0, max_retry=8, canceled=False): task = JobSchedule() task.uuid = "task-1" @@ -38,15 +39,58 @@ def _task(status=JobSchedule.STATUS_NEW, retry=0, max_retry=8, canceled=False): return task +class _Store: + """The task row plus the two ways it gets written, modelled faithfully. + + ``atomic_update`` follows DBController's contract: the mutator runs against + the row as it exists *in the store*, not against the caller's copy; a + mutator returning False aborts the write; the return is the fresh object, + or None when the row is gone. ``full_write`` is the ``write_to_db`` path — + it replaces the row wholesale from the caller's (possibly stale) copy. + """ + + def __init__(self, task): + self.kv_store = "KV" + self._rows = {task.uuid: copy.deepcopy(task)} + + # DBController surface used by the driver + def get_task_by_id(self, uuid): + row = self._rows.get(uuid) + return copy.deepcopy(row) if row is not None else None + + def atomic_update(self, obj, mutate): + row = self._rows.get(obj.uuid) + if row is None: + return None + fresh = copy.deepcopy(row) + if mutate(fresh) is False: + return copy.deepcopy(row) + self._rows[obj.uuid] = fresh + return copy.deepcopy(fresh) + + # test surface + def full_write(self, obj): + self._rows[obj.uuid] = copy.deepcopy(obj) + + def row(self, uuid="task-1"): + return self._rows[uuid] + + def concurrently(self, uuid="task-1", **fields): + """Another actor commits to the row while the handler is running.""" + for name, value in fields.items(): + setattr(self._rows[uuid], name, value) + + def _wire(monkeypatch, task, claim=True): - """Mock the module ``db`` (re-fetch returns the same task), the lease, and - the lease heartbeat (a no-op context manager so no real thread is spawned).""" - db = MagicMock() - db.get_task_by_id.return_value = task - monkeypatch.setattr(trb, "db", db) + """Point the driver at a fake store, grant the lease, and neutralise the + heartbeat (a no-op context manager, so no real thread is spawned).""" + store = _Store(task) + monkeypatch.setattr(trb, "db", store) + monkeypatch.setattr(JobSchedule, "write_to_db", + lambda self, kv=None: store.full_write(self)) monkeypatch.setattr(trb.tasks_controller, "claim_task", lambda *a, **k: claim) monkeypatch.setattr(trb.tasks_controller, "task_lease_heartbeat", MagicMock()) - return db + return store def _runner(handler, **spec_kw): @@ -58,11 +102,13 @@ def _runner(handler, **spec_kw): def test_void_return_marks_done(monkeypatch): task = _task() - _wire(monkeypatch, task) + store = _wire(monkeypatch, task) handler = MagicMock(return_value=None) + _runner(handler)._process(task, MagicMock()) - handler.assert_called_once_with(task) - assert task.status == JobSchedule.STATUS_DONE + + assert handler.call_args[0][0].uuid == "task-1" + assert store.row().status == JobSchedule.STATUS_DONE def test_handler_runs_under_lease_heartbeat(monkeypatch): @@ -70,86 +116,93 @@ def test_handler_runs_under_lease_heartbeat(monkeypatch): _wire(monkeypatch, task) hb = MagicMock() monkeypatch.setattr(trb.tasks_controller, "task_lease_heartbeat", hb) + _runner(MagicMock(return_value=None))._process(task, MagicMock()) - hb.assert_called_once_with(task) + + assert hb.call_args[0][0].uuid == "task-1" hb.return_value.__enter__.assert_called_once() hb.return_value.__exit__.assert_called_once() def test_defer_suspends_without_consuming_retry(monkeypatch): task = _task(retry=2) - _wire(monkeypatch, task) + store = _wire(monkeypatch, task) def handler(_task): raise trb.TaskDefer("node not online") runner = _runner(handler) runner._process(task, MagicMock()) - assert task.status == JobSchedule.STATUS_SUSPENDED - assert task.retry == 2 - assert task.function_result == "node not online" + + assert store.row().status == JobSchedule.STATUS_SUSPENDED + assert store.row().retry == 2 + assert store.row().function_result == "node not online" assert "task-1" not in runner._next_attempt # no backoff for a defer def test_retry_suspends_consumes_retry_and_backs_off(monkeypatch): task = _task(retry=1) - _wire(monkeypatch, task) + store = _wire(monkeypatch, task) def handler(_task): raise trb.TaskRetry("rpc failed") runner = _runner(handler) runner._process(task, MagicMock()) - assert task.status == JobSchedule.STATUS_SUSPENDED - assert task.retry == 2 + + assert store.row().status == JobSchedule.STATUS_SUSPENDED + assert store.row().retry == 2 assert runner._next_attempt["task-1"] > time.time() def test_unexpected_exception_is_treated_as_retry(monkeypatch): task = _task(retry=0) - _wire(monkeypatch, task) + store = _wire(monkeypatch, task) def handler(_task): raise RuntimeError("boom") - runner = _runner(handler) - runner._process(task, MagicMock()) # must not raise - assert task.status == JobSchedule.STATUS_SUSPENDED - assert task.retry == 1 + _runner(handler)._process(task, MagicMock()) # must not raise + + assert store.row().status == JobSchedule.STATUS_SUSPENDED + assert store.row().retry == 1 def test_success_message_comes_from_the_handler(monkeypatch): task = _task() - _wire(monkeypatch, task) + store = _wire(monkeypatch, task) def handler(t): t.function_result = "Backup created" _runner(handler)._process(task, MagicMock()) - assert task.status == JobSchedule.STATUS_DONE - assert task.function_result == "Backup created" + + assert store.row().status == JobSchedule.STATUS_DONE + assert store.row().function_result == "Backup created" def test_previous_failure_result_does_not_survive_a_later_success(monkeypatch): task = _task(status=JobSchedule.STATUS_SUSPENDED, retry=1) task.function_result = "rpc failed" - _wire(monkeypatch, task) + store = _wire(monkeypatch, task) _runner(MagicMock(return_value=None))._process(task, MagicMock()) - assert task.status == JobSchedule.STATUS_DONE - assert task.function_result == "completed" + + assert store.row().status == JobSchedule.STATUS_DONE + assert store.row().function_result == "completed" def test_abort_marks_done_with_reason(monkeypatch): task = _task() - _wire(monkeypatch, task) + store = _wire(monkeypatch, task) def handler(_task): raise trb.TaskAbort("missing param") _runner(handler)._process(task, MagicMock()) - assert task.status == JobSchedule.STATUS_DONE - assert task.function_result == "missing param" + + assert store.row().status == JobSchedule.STATUS_DONE + assert store.row().function_result == "missing param" # -- terminal cleanup hook -------------------------------------------------- @@ -160,14 +213,14 @@ def handler(_task): ]) def test_on_finish_runs_for_every_terminal_outcome(monkeypatch, handler, expected): task = _task() - _wire(monkeypatch, task) + store = _wire(monkeypatch, task) on_finish = MagicMock() _runner(handler, on_finish=on_finish)._process(task, MagicMock()) - on_finish.assert_called_once_with(task) - assert task.status == JobSchedule.STATUS_DONE - assert task.function_result == expected + assert on_finish.call_args[0][0].uuid == "task-1" + assert store.row().status == JobSchedule.STATUS_DONE + assert store.row().function_result == expected def test_on_finish_runs_when_the_handler_is_never_reached(monkeypatch): @@ -176,7 +229,8 @@ def test_on_finish_runs_when_the_handler_is_never_reached(monkeypatch): on_finish = MagicMock() _runner(MagicMock(), on_finish=on_finish)._process(task, MagicMock()) - on_finish.assert_called_once_with(task) + + on_finish.assert_called_once() def test_on_finish_does_not_run_for_a_suspended_task(monkeypatch): @@ -186,24 +240,26 @@ def test_on_finish_does_not_run_for_a_suspended_task(monkeypatch): _runner(MagicMock(side_effect=trb.TaskDefer("later")), on_finish=on_finish)._process(task, MagicMock()) + on_finish.assert_not_called() def test_failing_on_finish_does_not_break_the_task(monkeypatch): task = _task() - _wire(monkeypatch, task) + store = _wire(monkeypatch, task) runner = _runner(MagicMock(return_value=None), on_finish=MagicMock(side_effect=RuntimeError("cleanup boom"))) runner._process(task, MagicMock()) # must not raise - assert task.status == JobSchedule.STATUS_DONE + + assert store.row().status == JobSchedule.STATUS_DONE # -- pre-run skip-gates ----------------------------------------------------- def test_ineligible_skips_without_claim_or_write(monkeypatch): task = _task() - _wire(monkeypatch, task) + store = _wire(monkeypatch, task) claim = MagicMock(return_value=True) monkeypatch.setattr(trb.tasks_controller, "claim_task", claim) handler = MagicMock() @@ -212,56 +268,230 @@ def test_ineligible_skips_without_claim_or_write(monkeypatch): handler.assert_not_called() claim.assert_not_called() - JobSchedule.write_to_db.assert_not_called() - assert task.status == JobSchedule.STATUS_NEW + assert store.row().status == JobSchedule.STATUS_NEW def test_default_eligible_runs(monkeypatch): task = _task() _wire(monkeypatch, task) handler = MagicMock(return_value=None) + _runner(handler)._process(task, MagicMock()) + handler.assert_called_once() def test_lease_denied_skips(monkeypatch): task = _task() - _wire(monkeypatch, task, claim=False) + store = _wire(monkeypatch, task, claim=False) handler = MagicMock() + _runner(handler)._process(task, MagicMock()) + handler.assert_not_called() - assert task.status == JobSchedule.STATUS_NEW + assert store.row().status == JobSchedule.STATUS_NEW # -- terminal pre-handler checks ------------------------------------------- def test_canceled_marks_done_without_handler(monkeypatch): task = _task(canceled=True) - _wire(monkeypatch, task) + store = _wire(monkeypatch, task) handler = MagicMock() + _runner(handler)._process(task, MagicMock()) + handler.assert_not_called() - assert task.status == JobSchedule.STATUS_DONE - assert task.function_result == "canceled" + assert store.row().status == JobSchedule.STATUS_DONE + assert store.row().function_result == "canceled" def test_max_retry_marks_done_without_handler(monkeypatch): task = _task(retry=8, max_retry=8) - _wire(monkeypatch, task) + store = _wire(monkeypatch, task) handler = MagicMock() + _runner(handler)._process(task, MagicMock()) + handler.assert_not_called() - assert task.status == JobSchedule.STATUS_DONE - assert "max retry" in task.function_result + assert store.row().status == JobSchedule.STATUS_DONE + assert "max retry" in store.row().function_result def test_negative_max_retry_is_unbounded(monkeypatch): task = _task(retry=100, max_retry=-1) - _wire(monkeypatch, task) + store = _wire(monkeypatch, task) handler = MagicMock(return_value=None) + _runner(handler)._process(task, MagicMock()) + handler.assert_called_once() # ceiling never binds for max_retry < 0 - assert task.status == JobSchedule.STATUS_DONE + assert store.row().status == JobSchedule.STATUS_DONE + + +# -- concurrent-writer safety ---------------------------------------------- +# +# A handler runs for minutes (node add, restart, migration) while the driver +# holds the task copy it fetched beforehand. Other actors write that row in the +# meantime — set_node_status(ONLINE) cancels restart tasks +# (tasks_controller.cancel_pending_node_restart_tasks), an operator cancels a +# task, another host's lease heartbeat stamps it. A full-object write of the +# stale copy silently reverts all of that: it is what un-canceled a task and +# wiped its owner lease in the 2026-07-29 double-restart incident, and it is +# why upstream converted the restart runner's task writes to atomic CAS. + +def test_defer_does_not_resurrect_a_concurrently_canceled_task(monkeypatch): + task = _task() + store = _wire(monkeypatch, task) + + def handler(_task): + store.concurrently(canceled=True, status=JobSchedule.STATUS_DONE, + function_result="canceled: node back online") + raise trb.TaskDefer("peer is restarting") + + _runner(handler)._process(task, MagicMock()) + + row = store.row() + assert row.canceled is True + assert row.status == JobSchedule.STATUS_DONE + assert row.function_result == "canceled: node back online" + + +def test_failure_does_not_resurrect_a_concurrently_canceled_task(monkeypatch): + task = _task() + store = _wire(monkeypatch, task) + + def handler(_task): + store.concurrently(canceled=True, status=JobSchedule.STATUS_DONE) + raise trb.TaskRetry("rpc failed") + + _runner(handler)._process(task, MagicMock()) + + assert store.row().canceled is True + assert store.row().status == JobSchedule.STATUS_DONE + + +def test_write_does_not_steal_a_lease_taken_during_the_handler(monkeypatch): + task = _task() + task.owner = "this-host" + store = _wire(monkeypatch, task) + + def handler(_task): + store.concurrently(owner="other-host") + + _runner(handler)._process(task, MagicMock()) + + assert store.row().owner == "other-host" + + +def test_retry_is_counted_on_the_fresh_row(monkeypatch): + task = _task(retry=1) + store = _wire(monkeypatch, task) + + def handler(_task): + store.concurrently(retry=5) + raise trb.TaskRetry("rpc failed") + + _runner(handler)._process(task, MagicMock()) + + assert store.row().retry == 6 + + +def test_handler_progress_is_carried_onto_the_fresh_row(monkeypatch): + """Handlers record progress in function_params (recovery_started, + merge_started, fail_count) — a CAS that only wrote the lifecycle fields + would drop it and the next attempt would re-issue the RPC.""" + task = _task() + store = _wire(monkeypatch, task) + + def handler(t): + t.function_params["recovery_started"] = True + raise trb.TaskDefer("Restore started") + + _runner(handler)._process(task, MagicMock()) + + assert store.row().function_params["recovery_started"] is True + + +def test_on_finish_is_skipped_when_another_actor_finished_the_task(monkeypatch): + """Cleanup is a side effect of winning the terminal transition. Running it + off a lost CAS means two hosts both release the resource.""" + task = _task() + store = _wire(monkeypatch, task) + on_finish = MagicMock() + + def handler(_task): + store.concurrently(status=JobSchedule.STATUS_DONE, + function_result="canceled: node back online") + + _runner(handler, on_finish=on_finish)._process(task, MagicMock()) + + on_finish.assert_not_called() + assert store.row().function_result == "canceled: node back online" + + +# -- single dispatch path --------------------------------------------------- +# +# The 2026-07-29 incident's other half: restart had a parallel branch that +# consulted the inflight map and an inline branch that consulted neither, so a +# dispatch-mode flip mid-restart re-entered a task still running on the pool. +# Serial execution must therefore register in-flight exactly like parallel +# execution does. + +def test_a_running_task_is_not_re_entered_by_a_second_dispatch(monkeypatch): + task = _task() + store = _wire(monkeypatch, task) + calls, started, release = [], threading.Event(), threading.Event() + + def handler(_task): + calls.append(1) + started.set() + release.wait(5) + + runner = _runner(handler) + cluster = MagicMock() + + worker = threading.Thread(target=runner._dispatch, args=(task, cluster)) + worker.start() + try: + assert started.wait(5) + # The dispatch loop comes round again while the task is still running. + runner._dispatch(store.get_task_by_id("task-1"), cluster) + finally: + release.set() + worker.join(5) + + assert calls == [1] + + +def test_serialized_dispatch_waits_for_the_task(monkeypatch): + """Serialized mode must submit and wait, not run inline — that is what + makes a mode flip harmless in both directions.""" + task = _task() + store = _wire(monkeypatch, task) + + runner = _runner(MagicMock(return_value=None), concurrency=4, + serialize=lambda t, c: True) + runner._dispatch(task, MagicMock()) + + assert store.row().status == JobSchedule.STATUS_DONE + + +def test_parallel_dispatch_does_not_wait(monkeypatch): + task = _task() + _wire(monkeypatch, task) + started, release = threading.Event(), threading.Event() + + def handler(_task): + started.set() + release.wait(5) + + runner = _runner(handler, concurrency=4) + try: + runner._dispatch(task, MagicMock()) # returns while the handler runs + assert started.wait(5) + finally: + release.set() # -- dispatch loop ---------------------------------------------------------- From 844071daeb2716d8b4620f239e1836a7ef145ccf Mon Sep 17 00:00:00 2001 From: Max Schettler Date: Wed, 5 Aug 2026 14:20:12 +0200 Subject: [PATCH 16/25] Revise plan for the upstream restart hardening Records why B0's write model and dispatch path changed, defers the two runners upstream is actively rewriting, and moves restart to the front of the remainder since it is what the design has to satisfy. --- TASK_RUNNER_REWORK_PLAN.md | 99 +++++++++++++++++++++++++++++--------- 1 file changed, 75 insertions(+), 24 deletions(-) diff --git a/TASK_RUNNER_REWORK_PLAN.md b/TASK_RUNNER_REWORK_PLAN.md index 76a9218fa2..31ebc544fb 100644 --- a/TASK_RUNNER_REWORK_PLAN.md +++ b/TASK_RUNNER_REWORK_PLAN.md @@ -197,39 +197,90 @@ new module + unit tests, no runner changed yet. Provides: - `TaskAbort(reason)` → permanent / non-retryable (missing param, object not found, "not needed") → `STATUS_DONE` with a failure/short-circuit result. Replaces the handlers that currently mark DONE mid-body. -- Execution: `concurrency == 1` runs inline (serial); `> 1` uses a - `ThreadPoolExecutor` with a per-task inflight set and an optional - `exclusion_key(task)` per-key inflight set + capped exponential backoff — - generalizing `node_add`/`restart`. +- **Task writes are compare-and-set, never full-object writes** (revised after + the 2026-08 rebase — see "Upstream reconciliation" below). Every transition + runs as a mutator against the row as it stands (`db.atomic_update`), refusing + a row another actor has finished — and, for non-terminal transitions, one it + has canceled — and reporting whether it won. Only the two handler-owned + fields (`function_result`, `function_params`) are carried over from the + driver's copy. `on_finish` runs only for the winner of the terminal + transition. +- Execution: **one dispatch path**. Every execution is submitted to the pool and + registered in the per-task inflight set (plus the optional + `exclusion_key(task)` per-key set); *serialized* execution submits and waits + on the future rather than running inline. Capped exponential backoff on + failure. Generalizes `node_add`/`restart`. - A `RunnerSpec` describing: `function_names`, `handler` (a void callable), `is_eligible` (default `lambda task, cluster: True`), `interval` (each runner - keeps its current cadence), `concurrency`, `exclusion_key`. + keeps its current cadence), `concurrency`, `exclusion_key`, `on_finish`, + `serialize` (per-task/per-cycle predicate; defaults to `concurrency == 1`). **B1…Bn — Migrate one runner per commit**, simplest → hardest, each preserving behavior and keeping the module-level handler importable for the existing tests: -1. `fdb_backup`, `jc_comp`, `replication_final`, `sync_lvol_del` (trivial serial) -2. `backup`, `cluster_expand` -3. `node_add` (concurrency opt-in, no exclusion key) -4. migration trio — `migration`, `new_dev_migration`, `failed_migration` +1. ✅ `fdb_backup`, `jc_comp`, `replication_final`, `sync_lvol_del` (trivial serial) +2. ✅ `backup`, `cluster_expand` +3. ✅ `node_add` (concurrency opt-in, `node_addr` exclusion key) +4. `restart` — **moved up** (was 6th). Concurrency + per-node exclusion; its + parallel-vs-serialized choice is the spec's `serialize` predicate + (`suspend_drain_complete` / `fd_dead_recovery_allowed`), and + `is_auto_restart_paused` becomes part of `is_eligible`. The driver's + `exclusion_key` + backoff subsumes `_node_inflight`/`_restart_next_attempt`, + and its `_task_finish`/`_task_update` CAS helpers are subsumed by the + driver's. Migrated first of the remainder because it is the runner that + defines the driver's hard requirements — if the design holds here it holds + everywhere. +5. migration trio — `migration`, `new_dev_migration`, `failed_migration` (serial; `get_active_node_mig_task` / same-node-sibling gating becomes the spec's `is_eligible` predicate, not a concurrency `exclusion_key`) -5. `port_allow` (large; its `is_eligible` omits the `IN_ACTIVATION` check — the +6. `port_allow` (large; its `is_eligible` omits the `IN_ACTIVATION` check — the documented opt-out — and keeps the recovery logic) -6. `restart` (concurrency + per-node exclusion + `fd_dead_recovery_allowed` / - `is_auto_restart_paused` gating; the driver's `exclusion_key` + backoff - subsumes `_node_inflight`/`_restart_next_attempt`. `is_auto_restart_paused` - becomes part of `is_eligible`) -7. `lvol_migration` (largest; migrate the loop/lease/retry shell only, leave the - domain snapshot-copy state machine intact. Its pre-run guard chain — node - status, `cluster.status in (ACTIVE, DEGRADED)`, open cluster-expand task — - becomes the spec's `is_eligible` predicate; unbounded `max_retry=-1` is kept) -8. `node_removal` (new upstream runner, `FN_NODE_REMOVAL`; already leased, - module-level loop, unbounded — a straightforward serial migration) -9. `batch_migration` (new upstream runner, `FN_LVOL_BATCH_MIG`, ~884 lines; - migrate the loop/lease shell, leave its group orchestration intact) - -**Upstream reconciliation (done during the 2026-07 rebase onto origin/main).** +7. `node_removal` (`FN_NODE_REMOVAL`; already leased, module-level loop, + unbounded — a straightforward serial migration) + +**Deferred to a follow-up PR** (see rebase-friendliness below — both are under +active upstream development, and neither blocks the rest): + +- `lvol_migration` (largest; migrate the loop/lease/retry shell only, leave the + domain snapshot-copy state machine intact. Its pre-run guard chain — node + status, `cluster.status in (ACTIVE, DEGRADED)`, open cluster-expand task — + becomes the spec's `is_eligible` predicate; unbounded `max_retry=-1` is kept) +- `batch_migration` (`FN_LVOL_BATCH_MIG`; migrate the loop/lease shell, leave + its group orchestration intact) + +**Upstream reconciliation (2026-08 rebase onto origin/main `4cb16a20b`).** +Upstream landed `a61b00ad4` — *"fix(restart): stop double execution of a +node-restart task (2026-07-29 incident)"* — which invalidates two of B0's +original choices, both now corrected above: + +- **Full-object task writes are unsafe.** A `task.write_to_db()` of a copy read + before a long handler ran reverts whatever other actors committed meanwhile: + it un-canceled a task that `cancel_pending_node_restart_tasks` canceled when + the node came back ONLINE, and reclaimed a lease another host had taken. The + driver held that stale copy for the *entire* handler duration, i.e. the widest + possible window, and centralizing it would have propagated the defect to every + runner. All transitions are CAS now; the migrated runners needed no changes, + since none of them touch task state. +- **A split dispatch path re-enters running tasks.** Restart chooses parallel vs + serialized per task per cycle from live cluster state (`suspend_drain_complete`, + `fd_dead_recovery_allowed`), and a mode flip mid-restart re-entered a task + still running on the pool because the inline branch consulted no inflight map. + Hence the single dispatch path and the `serialize` predicate: `concurrency` + cannot be a static number for restart. + +Also revised: B0 originally said handlers stop re-fetching. They stop +re-fetching for *lifecycle* decisions, but a handler must re-read the task +immediately before a destructive step — upstream added exactly that before +restart's shutdown. + +**Rebase-friendliness (2026-08).** `lvol_migration` and `batch_migration` took +~17 upstream commits in this window (cleanup state machine rewritten, multipath, +retry ceilings removed), several still labelled `TEMP:`. B7/B9 are therefore +deferred to a follow-up PR rather than rebased repeatedly against a moving +target; `restart` moves up to be migrated first, since it is the runner that +defines the driver's hard requirements. + +**Earlier upstream reconciliation (2026-07 rebase).** Upstream independently reworked the runners: renamed `task_runner`→`process_task` and added `tasks_controller.task_lease_heartbeat` (a lease-keepalive thread) to the six long-blocking runners. This is now **folded into the B0 driver** — the From efa72dabf39b59ea2e949f51d3c563be2a33585f Mon Sep 17 00:00:00 2001 From: Max Schettler Date: Wed, 5 Aug 2026 16:34:17 +0200 Subject: [PATCH 17/25] Cancel tasks by compare-and-set MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both cancellation paths wrote the whole task object from a copy read before they decided to cancel — cancel_pending_node_restart_tasks off a bulk get_job_tasks scan, so stale by construction. The runner driving that task writes the same row meanwhile, and the write put all of it back: the owner lease cleared (handing the task to the next runner host that polls, which runs it again), a finished task reverted to running, retry and handler progress rolled back. This is the same lost update as the 2026-07-29 double restart, arriving from the canceller's side rather than the runner's, so it is fixed the same way: a mutator against the current row. cancel_task now sets only the flag and leaves the task where the runner had got to, which reads it on its next pass. It no longer emits a second task-canceled event for an already-canceled task. cancel_pending_node_restart_tasks re-checks on the fresh row and leaves a task that reached its own outcome in between alone, rather than overwriting that outcome with "canceled: node back online". --- .../controllers/tasks_controller.py | 68 +++++- tests/unit/test_task_cancellation.py | 219 ++++++++++++++++++ 2 files changed, 280 insertions(+), 7 deletions(-) create mode 100644 tests/unit/test_task_cancellation.py diff --git a/simplyblock_core/controllers/tasks_controller.py b/simplyblock_core/controllers/tasks_controller.py index 455b6c0c39..a4192bf0be 100644 --- a/simplyblock_core/controllers/tasks_controller.py +++ b/simplyblock_core/controllers/tasks_controller.py @@ -6,6 +6,7 @@ import threading import time import uuid +from typing import Callable, Optional from simplyblock_core import db_controller, constants, utils from simplyblock_core.controllers import tasks_events, device_controller @@ -94,6 +95,42 @@ def _mutate(t): return refreshed["ok"] +def _cancel_atomically( + task: JobSchedule, + mutate: Callable[[JobSchedule], bool], +) -> Optional[JobSchedule]: + """Apply a cancellation to the task row as it currently stands. + + NOT ``task.write_to_db()``: the copy a canceller holds was read before it + decided to cancel — off a bulk ``get_job_tasks`` scan, in the case below — + and the runner driving that task writes the same row meanwhile, claiming + its lease, moving it to running, advancing retry and recording handler + progress in function_params. A full-object write puts all of that back. The + damaging one is the owner lease: clearing it hands the task to the next + runner host that polls, which executes it a second time. That is the lost + update behind the 2026-07-29 double restart, arriving from the other side. + + ``mutate`` receives the fresh row and returns False to decline (a guard + that no longer holds). It may be replayed on transaction conflict, so it + must do nothing but mutate the object it is given. + + Returns the committed task if this call performed the cancellation, or None + if the row is gone or another actor got there first. + """ + now = str(datetime.datetime.now(datetime.timezone.utc)) + performed = {"ok": False} + + def _mutate(fresh): + if mutate(fresh) is False: + return False + fresh.updated_at = now + performed["ok"] = True + return True + + committed = db.atomic_update(task, _mutate) + return committed if performed["ok"] else None + + @contextlib.contextmanager def task_lease_heartbeat(task, owner=None): """Refresh this host's lease on `task` every TASK_LEASE_HEARTBEAT_SEC for @@ -530,16 +567,25 @@ def cancel_pending_node_restart_tasks(cluster_id, node_id): # via the dedup guard in `_validate_new_task_node_restart` until the # task runner happens to pick it up — observed as a 5-minute window # of failing manual restarts after the node was already back online. + def _cancel_if_still_pending(fresh: JobSchedule) -> bool: + # Re-checked on the fresh row: a task that reached its own outcome + # between the scan and this write has nothing left to cancel, and its + # result must not be overwritten with ours. + if fresh.canceled or fresh.status == JobSchedule.STATUS_DONE: + return False + fresh.canceled = True + fresh.status = JobSchedule.STATUS_DONE + fresh.function_result = "canceled: node back online" + return True + canceled = 0 for task in db.get_job_tasks(cluster_id): if (task.function_name == JobSchedule.FN_NODE_RESTART and task.node_id == node_id and task.status != JobSchedule.STATUS_DONE and not task.canceled): - task.canceled = True - task.status = JobSchedule.STATUS_DONE - task.function_result = "canceled: node back online" - task.write_to_db(db.kv_store) + if _cancel_atomically(task, _cancel_if_still_pending) is None: + continue canceled += 1 logger.info( f"Canceled obsolete node_restart task {task.get_id()} (node {node_id} back online)") @@ -618,9 +664,17 @@ def cancel_task(task_id): if task.device_id: device_controller.device_set_retries_exhausted(task.device_id, True) - task.canceled = True - task.write_to_db(db.kv_store) - tasks_events.task_canceled(task) + def _flag_canceled(fresh): + if fresh.canceled: + return False + fresh.canceled = True + + # Only the flag is set. Where the task has got to — status, retry, owner, + # progress — stays as the runner left it; the runner reads the flag on its + # next pass and finishes the task itself. + committed = _cancel_atomically(task, _flag_canceled) + if committed is not None: + tasks_events.task_canceled(committed) return True diff --git a/tests/unit/test_task_cancellation.py b/tests/unit/test_task_cancellation.py new file mode 100644 index 0000000000..009b94a16c --- /dev/null +++ b/tests/unit/test_task_cancellation.py @@ -0,0 +1,219 @@ +# coding=utf-8 +"""Unit tests for task cancellation (tasks_controller.cancel_task and +cancel_pending_node_restart_tasks). + +A canceller reads the task, decides, and writes — and in between, the runner +driving that task is writing the same row: claiming its lease, moving it to +running, advancing retry, recording handler progress. A full-object +``write_to_db`` of the canceller's copy puts all of that back. The worst of it +is the owner lease: wiping it hands the task to the next runner host that polls, +which then executes it a second time. That is the same class of lost update as +the 2026-07-29 double restart, in the opposite direction. + +These use a store that models both write paths, so the assertions are about what +ends up committed rather than about the caller's copy. +""" +import copy +from unittest.mock import MagicMock + +import pytest + +from simplyblock_core.controllers import tasks_controller +from simplyblock_core.models.job_schedule import JobSchedule + + +def _task(uuid="task-1", function_name=JobSchedule.FN_NODE_RESTART, + status=JobSchedule.STATUS_NEW, node_id="node-1"): + task = JobSchedule() + task.uuid = uuid + task.cluster_id = "cl-1" + task.node_id = node_id + task.function_name = function_name + task.status = status + task.canceled = False + task.retry = 0 + task.function_params = {} + return task + + +class _Store: + """Task rows plus the two ways they get written. + + ``atomic_update`` follows DBController's contract: the mutator runs against + the row as it exists in the store, a mutator returning False aborts the + write, and the return is the fresh object (or None when the row is gone). + ``full_write`` is the ``write_to_db`` path — a wholesale replace from the + caller's copy. + """ + + def __init__(self, *tasks): + self.kv_store = "KV" + self._rows = {t.uuid: copy.deepcopy(t) for t in tasks} + self._stale = None + + def get_task_by_id(self, task_id): + if self._stale is not None: + return copy.deepcopy(self._stale[task_id]) + if task_id not in self._rows: + raise KeyError(task_id) + return copy.deepcopy(self._rows[task_id]) + + def get_job_tasks(self, cluster_id, reverse=True, limit=0): + rows = self._stale if self._stale is not None else self._rows + return [copy.deepcopy(row) for row in rows.values()] + + def atomic_update(self, obj, mutate): + row = self._rows.get(obj.uuid) + if row is None: + return None + fresh = copy.deepcopy(row) + if mutate(fresh) is False: + return copy.deepcopy(row) + self._rows[obj.uuid] = fresh + return copy.deepcopy(fresh) + + def full_write(self, obj): + self._rows[obj.uuid] = copy.deepcopy(obj) + + def row(self, uuid="task-1"): + return self._rows[uuid] + + def runner_claims(self, uuid="task-1", **fields): + """The runner drives the task after the canceller has read it: pin what + the canceller saw, then advance the row underneath it.""" + if self._stale is None: + self._stale = copy.deepcopy(self._rows) + row = self._rows[uuid] + row.owner = fields.pop("owner", "host-A") + row.status = fields.pop("status", JobSchedule.STATUS_RUNNING) + for name, value in fields.items(): + setattr(row, name, value) + + +@pytest.fixture +def store(monkeypatch): + def _install(*tasks): + s = _Store(*tasks) + monkeypatch.setattr(tasks_controller, "db", s) + monkeypatch.setattr(JobSchedule, "write_to_db", + lambda self, kv=None: s.full_write(self)) + return s + monkeypatch.setattr(tasks_controller, "tasks_events", MagicMock()) + monkeypatch.setattr(tasks_controller, "device_controller", MagicMock()) + return _install + + +# -- cancel_task ------------------------------------------------------------ + +def test_cancel_task_does_not_wipe_the_lease_of_a_running_task(store): + """The lease is what stops a second host from running the task. A cancel + that clears it hands the task straight to the next poller.""" + s = store(_task()) + s.runner_claims(owner="host-A", retry=1) + + assert tasks_controller.cancel_task("task-1") is True + + row = s.row() + assert row.canceled is True + assert row.owner == "host-A" + assert row.status == JobSchedule.STATUS_RUNNING + assert row.retry == 1 + + +def test_cancel_task_does_not_revert_a_finished_task(store): + s = store(_task()) + s.runner_claims(owner="host-A", status=JobSchedule.STATUS_DONE, + function_result="Node is online") + + tasks_controller.cancel_task("task-1") + + row = s.row() + assert row.status == JobSchedule.STATUS_DONE + assert row.function_result == "Node is online" + + +def test_cancel_task_does_not_lose_handler_progress(store): + """function_params carries multi-cycle progress (recovery_started, + merge_started); reverting it makes the next attempt re-issue the RPC.""" + s = store(_task(function_name=JobSchedule.FN_BACKUP_RESTORE)) + s.runner_claims(owner="host-A", function_params={"recovery_started": True}) + + tasks_controller.cancel_task("task-1") + + assert s.row().function_params == {"recovery_started": True} + assert s.row().canceled is True + + +def test_cancel_task_flags_and_emits_the_event(store): + s = store(_task()) + + assert tasks_controller.cancel_task("task-1") is True + + assert s.row().canceled is True + tasks_controller.tasks_events.task_canceled.assert_called_once() + + +def test_cancel_task_is_idempotent(store): + s = store(_task()) + tasks_controller.cancel_task("task-1") + tasks_controller.tasks_events.task_canceled.reset_mock() + + assert tasks_controller.cancel_task("task-1") is True + + assert s.row().canceled is True + tasks_controller.tasks_events.task_canceled.assert_not_called() + + +def test_cancel_task_refuses_a_master_task(store): + task = _task() + task.sub_tasks = ["sub-1"] + s = store(task) + + assert tasks_controller.cancel_task("task-1") is False + assert s.row().canceled is False + + +# -- cancel_pending_node_restart_tasks -------------------------------------- + +def test_cancel_pending_restart_does_not_revert_concurrent_progress(store): + """Called from set_node_status the moment a node goes ONLINE, off a bulk + read — so its copies are stale by construction.""" + s = store(_task()) + s.runner_claims(owner="host-A", retry=3) + + assert tasks_controller.cancel_pending_node_restart_tasks("cl-1", "node-1") == 1 + + row = s.row() + assert row.canceled is True + assert row.status == JobSchedule.STATUS_DONE + assert row.function_result == "canceled: node back online" + assert row.owner == "host-A" + assert row.retry == 3 + + +def test_cancel_pending_restart_leaves_a_finished_task_alone(store): + """A task that reached its own outcome between the bulk read and the write + keeps that outcome — the cancellation is moot.""" + s = store(_task()) + s.runner_claims(status=JobSchedule.STATUS_DONE, + function_result="Node is online") + + assert tasks_controller.cancel_pending_node_restart_tasks("cl-1", "node-1") == 0 + + row = s.row() + assert row.function_result == "Node is online" + assert row.canceled is False + + +def test_cancel_pending_restart_skips_other_nodes_and_functions(store): + s = store( + _task(uuid="task-1", node_id="node-1"), + _task(uuid="task-2", node_id="node-2"), + _task(uuid="task-3", node_id="node-1", function_name=JobSchedule.FN_NODE_ADD), + ) + + assert tasks_controller.cancel_pending_node_restart_tasks("cl-1", "node-1") == 1 + + assert s.row("task-1").canceled is True + assert s.row("task-2").canceled is False + assert s.row("task-3").canceled is False From 2cd55bf6f4401c43033b3190acb6881a02baa8fa Mon Sep 17 00:00:00 2001 From: Max Schettler Date: Wed, 5 Aug 2026 17:21:06 +0200 Subject: [PATCH 18/25] Cancel a node's tasks by compare-and-set on shutdown MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The last full-object cancellation write: node shutdown flagged the node's migration tasks canceled from a bulk get_job_tasks scan, wiping the owner lease of any the migration runner was driving — the same double-execution vector just fixed in the other two paths. It moves behind tasks_controller.cancel_node_tasks so all three cancellers share one CAS. This one declines a task that finished between the scan and the write, where an operator's explicit cancel_task stays deliberately unconditional. --- .../controllers/tasks_controller.py | 51 ++++++++++++++++--- simplyblock_core/storage_node_ops.py | 16 ++---- tests/unit/test_task_cancellation.py | 48 +++++++++++++++++ 3 files changed, 96 insertions(+), 19 deletions(-) diff --git a/simplyblock_core/controllers/tasks_controller.py b/simplyblock_core/controllers/tasks_controller.py index a4192bf0be..ba81b08b07 100644 --- a/simplyblock_core/controllers/tasks_controller.py +++ b/simplyblock_core/controllers/tasks_controller.py @@ -131,6 +131,26 @@ def _mutate(fresh): return committed if performed["ok"] else None +def _flag_canceled(fresh: JobSchedule) -> bool: + """Set only the canceled flag. Where the task has got to — status, retry, + owner, progress — stays as the runner left it; the runner reads the flag on + its next pass and finishes the task itself.""" + if fresh.canceled: + return False + fresh.canceled = True + return True + + +def _flag_canceled_if_pending(fresh: JobSchedule) -> bool: + """As _flag_canceled, but declines a task that has already finished. An + opportunistic bulk cancellation has nothing left to cancel there, and + flagging it would misreport a completed task as canceled. (An operator's + explicit cancel_task is deliberately not this strict.)""" + if fresh.status == JobSchedule.STATUS_DONE: + return False + return _flag_canceled(fresh) + + @contextlib.contextmanager def task_lease_heartbeat(task, owner=None): """Refresh this host's lease on `task` every TASK_LEASE_HEARTBEAT_SEC for @@ -592,6 +612,29 @@ def _cancel_if_still_pending(fresh: JobSchedule) -> bool: return canceled +def cancel_node_tasks(cluster_id, node_id, function_names): + """Flag the node's unfinished tasks of the given kinds canceled. + + Used when a node is going away and the work queued against it is moot + (shutdown cancelling its migration tasks). Like every canceller this reads + in bulk and writes one row at a time, so each write goes through the CAS in + _cancel_atomically rather than putting the scan's copy back. + + Returns how many tasks this call canceled. + """ + canceled = 0 + for task in db.get_job_tasks(cluster_id): + if task.node_id != node_id or task.function_name not in function_names: + continue + if task.status == JobSchedule.STATUS_DONE or task.canceled: + continue + if _cancel_atomically(task, _flag_canceled_if_pending) is None: + continue + canceled += 1 + logger.info(f"Canceled task {task.get_id()} ({task.function_name}) on node {node_id}") + return canceled + + def list_tasks(cluster_id, is_json=False, limit=50, **kwargs): try: db.get_cluster_by_id(cluster_id) @@ -664,14 +707,6 @@ def cancel_task(task_id): if task.device_id: device_controller.device_set_retries_exhausted(task.device_id, True) - def _flag_canceled(fresh): - if fresh.canceled: - return False - fresh.canceled = True - - # Only the flag is set. Where the task has got to — status, retry, owner, - # progress — stays as the runner left it; the runner reads the flag on its - # next pass and finishes the task itself. committed = _cancel_atomically(task, _flag_canceled) if committed is not None: tasks_events.task_canceled(committed) diff --git a/simplyblock_core/storage_node_ops.py b/simplyblock_core/storage_node_ops.py index 654cc8b405..167f5f0ff2 100644 --- a/simplyblock_core/storage_node_ops.py +++ b/simplyblock_core/storage_node_ops.py @@ -5986,17 +5986,11 @@ def shutdown_storage_node(node_id, force=False, keep_auto_restart=False, snode.write_to_db(db_controller.kv_store) # Step 2: cancel migration tasks while controllers are still up. - pending_tasks = db_controller.get_job_tasks(snode.cluster_id) - for task in pending_tasks: - if task.node_id != node_id or task.status == JobSchedule.STATUS_DONE: - continue - if task.function_name in [ - JobSchedule.FN_DEV_MIG, - JobSchedule.FN_FAILED_DEV_MIG, - JobSchedule.FN_NEW_DEV_MIG, - ]: - task.canceled = True - task.write_to_db(db_controller.kv_store) + tasks_controller.cancel_node_tasks(snode.cluster_id, node_id, [ + JobSchedule.FN_DEV_MIG, + JobSchedule.FN_FAILED_DEV_MIG, + JobSchedule.FN_NEW_DEV_MIG, + ]) if not force: # Step 3 (Loop 1): broadcast device-unavailable events. The diff --git a/tests/unit/test_task_cancellation.py b/tests/unit/test_task_cancellation.py index 009b94a16c..7946055e23 100644 --- a/tests/unit/test_task_cancellation.py +++ b/tests/unit/test_task_cancellation.py @@ -217,3 +217,51 @@ def test_cancel_pending_restart_skips_other_nodes_and_functions(store): assert s.row("task-1").canceled is True assert s.row("task-2").canceled is False assert s.row("task-3").canceled is False + + +# -- cancel_node_tasks ------------------------------------------------------ + +def test_cancel_node_tasks_does_not_wipe_the_lease(store): + """Node shutdown cancels the migration tasks queued against the node, off + the same kind of bulk read.""" + s = store(_task(function_name=JobSchedule.FN_DEV_MIG)) + s.runner_claims(owner="host-A", retry=2) + + assert tasks_controller.cancel_node_tasks( + "cl-1", "node-1", [JobSchedule.FN_DEV_MIG]) == 1 + + row = s.row() + assert row.canceled is True + assert row.owner == "host-A" + assert row.status == JobSchedule.STATUS_RUNNING + assert row.retry == 2 + + +def test_cancel_node_tasks_leaves_a_finished_task_alone(store): + s = store(_task(function_name=JobSchedule.FN_DEV_MIG)) + s.runner_claims(status=JobSchedule.STATUS_DONE, function_result="migrated") + + assert tasks_controller.cancel_node_tasks( + "cl-1", "node-1", [JobSchedule.FN_DEV_MIG]) == 0 + + assert s.row().canceled is False + assert s.row().function_result == "migrated" + + +def test_cancel_node_tasks_skips_other_nodes_and_functions(store): + s = store( + _task(uuid="task-1", node_id="node-1", function_name=JobSchedule.FN_DEV_MIG), + _task(uuid="task-2", node_id="node-1", function_name=JobSchedule.FN_NEW_DEV_MIG), + _task(uuid="task-3", node_id="node-2", function_name=JobSchedule.FN_DEV_MIG), + _task(uuid="task-4", node_id="node-1", function_name=JobSchedule.FN_NODE_RESTART), + ) + + assert tasks_controller.cancel_node_tasks( + "cl-1", "node-1", + [JobSchedule.FN_DEV_MIG, JobSchedule.FN_FAILED_DEV_MIG, + JobSchedule.FN_NEW_DEV_MIG]) == 2 + + assert s.row("task-1").canceled is True + assert s.row("task-2").canceled is True + assert s.row("task-3").canceled is False + assert s.row("task-4").canceled is False From bf55257a2297991d5556a1f16de393e8576c5a1d Mon Sep 17 00:00:00 2001 From: Max Schettler Date: Wed, 5 Aug 2026 20:39:45 +0200 Subject: [PATCH 19/25] Add handler checkpoints and a per-cycle hook to the task runner MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both are needed by the restart runner and generalize beyond it. checkpoint() lets a handler record progress the moment an expensive or destructive step succeeds, rather than when the handler returns: restart must not repeat its cleanup shutdown after a crash between the shutdown and the restart. It doubles as the cancellation probe such a handler needs before its next destructive step. on_cycle covers upkeep a runner owns that is attached to no task — restart's watchdog for nodes stranded in a transitional state with no task owning them. The CAS commit moves to a module-level function so both the driver's transitions and checkpoint() share it. --- simplyblock_core/services/task_runner_base.py | 132 ++++++++++++------ tests/unit/tasks/test_task_runner_base.py | 76 ++++++++++ 2 files changed, 162 insertions(+), 46 deletions(-) diff --git a/simplyblock_core/services/task_runner_base.py b/simplyblock_core/services/task_runner_base.py index 7d7335906f..556f12fd5d 100644 --- a/simplyblock_core/services/task_runner_base.py +++ b/simplyblock_core/services/task_runner_base.py @@ -83,6 +83,73 @@ def _default_eligible(task: JobSchedule, cluster: Any) -> bool: return True +def _commit(task: JobSchedule, apply: Callable[[JobSchedule], None], + terminal: bool = False, context: str = "task-runner") -> Optional[JobSchedule]: + """Commit a change onto the task row as it exists NOW. + + Never a full-object ``write_to_db`` of ``task``: that copy was read before + the handler ran, and a handler runs for minutes (node add, restart, + migration). Writing it back reinstates every field another actor changed + meanwhile — it un-cancels a task that ``cancel_pending_node_restart_tasks`` + canceled when the node came back ONLINE, and reclaims a lease another host + has since taken. That pair of lost updates is what re-ran a restart against + an already-recovered node (2026-07-29 double restart). + + Only the two fields a handler owns are carried over from ``task``: + ``function_result`` and ``function_params`` (where handlers record progress + like ``recovery_started`` / ``merge_started``, which must survive so the + next attempt does not repeat the step). + + A terminal commit may finish a canceled task — that IS the cancellation + being carried out; a non-terminal one would be reviving it. + + Returns the committed task, or None if another actor already owns the + outcome — the caller must stop driving it. + """ + result = task.function_result + params = task.function_params + now = str(datetime.datetime.now(datetime.timezone.utc)) + won = {"ok": False} + + def _mutate(fresh: JobSchedule): + if fresh.status == JobSchedule.STATUS_DONE: + return False + if not terminal and fresh.canceled: + return False + fresh.function_result = result + fresh.function_params = params + apply(fresh) + fresh.updated_at = now + won["ok"] = True + return True + + committed = db.atomic_update(task, _mutate) + if committed is None or not won["ok"]: + logger.info(f"{context}: task {task.uuid} was finished or canceled " + f"concurrently; another actor owns the outcome") + return None + return committed + + +def checkpoint(task: JobSchedule, **params) -> Optional[JobSchedule]: + """Record handler progress on the task, mid-handler. + + For a long handler with a step that must not be repeated — a cleanup + shutdown, an issued transfer — mark it done the moment it succeeds rather + than when the handler returns, where a crash in between would lose the fact + and repeat the step on the next attempt. + + Doubles as the cancellation probe such a handler needs anyway: returns the + fresh task to carry on with, or None if the task was canceled or finished + underneath it, in which case the handler must stop rather than proceed to + the next destructive step. + """ + def _apply(fresh: JobSchedule) -> None: + fresh.function_params = dict(fresh.function_params, **params) + + return _commit(task, _apply) + + @dataclass class RunnerSpec: """Describes one task runner. ``function_names`` and ``handler`` are the only @@ -115,6 +182,11 @@ class RunnerSpec: # restart fans out only for a drained suspension or a fully-dead failure # domain) supplies a predicate instead of a fixed concurrency. serialize: Optional[Callable[[JobSchedule, Any], bool]] = None + # Optional per-cluster work, run once each cycle after that cluster's tasks + # are dispatched. For upkeep a runner owns that is not attached to any task + # — the restart runner's watchdog for nodes left in a transitional state + # with no task owning them. Failures are logged, never fatal. + on_cycle: Optional[Callable[[Any], None]] = None def __post_init__(self) -> None: if self.concurrency < 1: @@ -156,8 +228,21 @@ def run(self) -> None: self._forget(task.uuid) continue self._dispatch(task, cl) + self._run_cycle_hook(cl) time.sleep(self.spec.interval) + def _run_cycle_hook(self, cluster: Any) -> None: + if self.spec.on_cycle is None: + return + # Upkeep failing must not stop the loop from serving tasks — but a DB + # error still propagates, since that means the process should exit. + try: + self.spec.on_cycle(cluster) + except Exception as e: # noqa: BLE001 - upkeep failure is not fatal + logger.error(f"{self.spec.name}: cycle hook failed for " + f"cluster {cluster.get_id()}: {e}") + logger.exception(e) + # -- dispatch ----------------------------------------------------------- def _dispatch(self, task: JobSchedule, cluster: Any) -> None: @@ -269,52 +354,7 @@ def _process(self, task: JobSchedule, cluster: Any) -> None: def _cas(self, task: JobSchedule, apply: Callable[[JobSchedule], None], terminal: bool = False) -> Optional[JobSchedule]: - """Commit a lifecycle transition onto the task row as it exists NOW. - - Never a full-object ``write_to_db`` of ``task``: the driver fetched that - copy before the handler ran, and a handler runs for minutes (node add, - restart, migration). Writing it back reinstates every field another - actor changed meanwhile — it un-cancels a task that - ``cancel_pending_node_restart_tasks`` canceled when the node came back - ONLINE, and reclaims a lease another host has since taken. That pair of - lost updates is what re-ran a restart against an already-recovered node - (2026-07-29 double restart). - - Only the two fields a handler owns are carried over from ``task``: - ``function_result`` and ``function_params`` (where handlers record - progress like ``recovery_started`` / ``merge_started``, which must - survive so the next attempt does not re-issue the RPC). - - Returns the committed task, or None if another actor already owns the - outcome — the caller must stop driving it. - """ - result = task.function_result - params = task.function_params - now = str(datetime.datetime.now(datetime.timezone.utc)) - won = {"ok": False} - - def _mutate(fresh: JobSchedule): - if fresh.status == JobSchedule.STATUS_DONE: - return False - # A terminal transition is allowed to finish a canceled task — - # that IS the cancellation being carried out. A non-terminal one - # would be reviving it. - if not terminal and fresh.canceled: - return False - fresh.function_result = result - fresh.function_params = params - apply(fresh) - fresh.updated_at = now - won["ok"] = True - return True - - committed = db.atomic_update(task, _mutate) - if committed is None or not won["ok"]: - logger.info( - f"{self.spec.name}: task {task.uuid} was finished or canceled " - f"concurrently; another actor owns the outcome") - return None - return committed + return _commit(task, apply, terminal=terminal, context=self.spec.name) @staticmethod def _to(status: str) -> Callable[[JobSchedule], None]: diff --git a/tests/unit/tasks/test_task_runner_base.py b/tests/unit/tasks/test_task_runner_base.py index bc0c22bc5d..a8563e4816 100644 --- a/tests/unit/tasks/test_task_runner_base.py +++ b/tests/unit/tasks/test_task_runner_base.py @@ -539,3 +539,79 @@ def test_run_propagates_db_error(monkeypatch): def test_spec_rejects_zero_concurrency(): with pytest.raises(ValueError): trb.RunnerSpec(function_names=("fn",), handler=MagicMock(), concurrency=0) + + +# -- handler progress checkpoints ------------------------------------------- + +def test_checkpoint_persists_progress_immediately(monkeypatch): + """A destructive step records itself the moment it succeeds, so a crash + before the handler returns does not repeat it.""" + task = _task() + store = _wire(monkeypatch, task) + + fresh = trb.checkpoint(store.get_task_by_id("task-1"), cleanup_shutdown_done=True) + + assert fresh.function_params["cleanup_shutdown_done"] is True + assert store.row().function_params["cleanup_shutdown_done"] is True + + +def test_checkpoint_keeps_existing_params(monkeypatch): + task = _task() + task.function_params = {"node_addr": "1.2.3.4:5000"} + store = _wire(monkeypatch, task) + + trb.checkpoint(store.get_task_by_id("task-1"), cleanup_shutdown_done=True) + + assert store.row().function_params == { + "node_addr": "1.2.3.4:5000", "cleanup_shutdown_done": True} + + +def test_checkpoint_reports_a_cancellation_under_the_handler(monkeypatch): + """The handler's cancellation probe: a task canceled mid-handler must not + go on to the next destructive step.""" + task = _task() + store = _wire(monkeypatch, task) + store.concurrently(canceled=True) + + assert trb.checkpoint(store.get_task_by_id("task-1"), step_done=True) is None + assert "step_done" not in store.row().function_params + + +def test_checkpoint_reports_a_task_finished_under_the_handler(monkeypatch): + task = _task() + store = _wire(monkeypatch, task) + store.concurrently(status=JobSchedule.STATUS_DONE) + + assert trb.checkpoint(store.get_task_by_id("task-1"), step_done=True) is None + + +# -- per-cluster cycle hook ------------------------------------------------- + +def test_cycle_hook_runs_once_per_cluster(monkeypatch): + cluster = MagicMock() + cluster.get_id.return_value = "cl-1" + db = MagicMock() + db.get_clusters.return_value = [cluster] + db.get_job_tasks.return_value = [] + monkeypatch.setattr(trb, "db", db) + monkeypatch.setattr(trb.time, "sleep", MagicMock(side_effect=_StopLoop)) + on_cycle = MagicMock() + + with pytest.raises(_StopLoop): + _runner(MagicMock(), on_cycle=on_cycle).run() + + on_cycle.assert_called_once_with(cluster) + + +def test_failing_cycle_hook_does_not_stop_the_loop(monkeypatch): + cluster = MagicMock() + cluster.get_id.return_value = "cl-1" + db = MagicMock() + db.get_clusters.return_value = [cluster] + db.get_job_tasks.return_value = [] + monkeypatch.setattr(trb, "db", db) + monkeypatch.setattr(trb.time, "sleep", MagicMock(side_effect=_StopLoop)) + + runner = _runner(MagicMock(), on_cycle=MagicMock(side_effect=RuntimeError("watchdog boom"))) + with pytest.raises(_StopLoop): # reached the sleep, i.e. the cycle completed + runner.run() From 70b97d7239633216594976984506ea3db93c1187 Mon Sep 17 00:00:00 2001 From: Max Schettler Date: Wed, 5 Aug 2026 20:54:21 +0200 Subject: [PATCH 20/25] Migrate restart runner onto the task runner MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The runner that motivated the driver's CAS writes and single dispatch path now uses them: _task_finish/_task_update, the thread pool, both inflight maps and the per-task backoff schedule are all the driver's, which is most of the 200 lines this removes. Its parallel-vs-serialized choice becomes the spec's serialize predicate, and the condition now has one definition — the dispatch loop and the handler's peer-exclusion pre-check computed it separately, and had they ever disagreed a task fanned out in parallel would have immediately deferred on the peers it was dispatched alongside. The give-up side effects move to on_finish, keyed off the task's own state rather than off having just written it: parking the node OFFLINE and re-queueing when the ceiling terminates a node task, exhausting a device's retries when its task is canceled. Side effects the handler can still reach stay in the handler. Two behaviours needed re-expressing rather than moving: - "stop, a restart is already in flight" was inferred from the task still being NEW/SUSPENDED, which distinguishes nothing now that the driver sets RUNNING before calling. It records the fact directly instead, which also covers an attempt that issued the restart and then died — that RESTARTING is ours to finish, not to defer to. - The cleanup shutdown's once-flag is written through the driver's checkpoint, keeping it persisted the moment the shutdown succeeds rather than when the handler returns. Shutdown and restart raising stay defers, as before: neither consumed a retry, and restart's give-up has side effects that a changed verdict would start triggering where it previously could not. --- simplyblock_core/services/task_runner_base.py | 7 + .../services/tasks_runner_restart.py | 592 ++++++------------ .../test_tasks_runner_restart_shortcircuit.py | 27 +- tests/unit/tasks/test_max_retry_semantics.py | 96 ++- tests/unit/tasks/test_retry_ceiling.py | 25 +- tests/unit/test_restart_claim.py | 34 +- 6 files changed, 306 insertions(+), 475 deletions(-) diff --git a/simplyblock_core/services/task_runner_base.py b/simplyblock_core/services/task_runner_base.py index 556f12fd5d..a638ed797c 100644 --- a/simplyblock_core/services/task_runner_base.py +++ b/simplyblock_core/services/task_runner_base.py @@ -187,6 +187,11 @@ class RunnerSpec: # — the restart runner's watchdog for nodes left in a transitional state # with no task owning them. Failures are logged, never fatal. on_cycle: Optional[Callable[[Any], None]] = None + # Optional delay-before-next-attempt for a task that consumed a retry, + # given the new retry count. Defaults to interval * 2**(retry-1), capped. + # A runner whose recovery curve is tuned to its own workload (node restart + # holds a steady lead-in cadence before backing off) supplies its own. + backoff: Optional[Callable[[int], float]] = None def __post_init__(self) -> None: if self.concurrency < 1: @@ -418,6 +423,8 @@ def _apply(fresh: JobSchedule) -> None: def _backoff_delay(self, retry: int) -> float: if retry <= 0: return 0.0 + if self.spec.backoff is not None: + return self.spec.backoff(retry) exp = min(retry - 1, 16) # guard the shift against absurd retry counts return min(self.spec.interval * (2 ** exp), _BACKOFF_CAP_SEC) diff --git a/simplyblock_core/services/tasks_runner_restart.py b/simplyblock_core/services/tasks_runner_restart.py index 374051ca85..d266f696b2 100644 --- a/simplyblock_core/services/tasks_runner_restart.py +++ b/simplyblock_core/services/tasks_runner_restart.py @@ -1,7 +1,5 @@ # coding=utf-8 -import datetime import time -from concurrent.futures import ThreadPoolExecutor from simplyblock_core import constants, db_controller, storage_node_ops, utils from simplyblock_core.controllers import device_controller, health_controller, tasks_controller @@ -9,6 +7,14 @@ from simplyblock_core.models.job_schedule import JobSchedule from simplyblock_core.models.nvme_device import NVMeDevice from simplyblock_core.models.storage_node import StorageNode +from simplyblock_core.services.task_runner_base import ( + RunnerSpec, + TaskAbort, + TaskDefer, + TaskRetry, + checkpoint, + serve, +) from simplyblock_core.snode_client import SNodeClientException @@ -20,6 +26,43 @@ utils.init_sentry_sdk() +def _parallel_restart_allowed(node): + """The two sanctioned cases for restarting nodes in parallel: a drained + suspension (full-cluster recovery — every node offline, no client IO), and + a fully-dead failure domain (no domain member ONLINE, so parallel recovery + cannot touch served IO). + + Decides both the dispatch mode and whether the peer-exclusion pre-check in + task_runner_node applies. The two must agree: a task fanned out in parallel + would otherwise immediately defer on the very peers it was dispatched + alongside. + """ + cluster = db.get_cluster_by_id(node.cluster_id) + if cluster.status == Cluster.STATUS_SUSPENDED and cluster.suspend_drain_complete: + return True + return storage_node_ops.fd_dead_recovery_allowed(db, node) + + +def _is_eligible(task, cluster): + """Suspend recovery: while a SUSPENDED cluster is still being drained to + all-offline, pause node restarts. Executing one now would fight the + auto-shutdown and re-create the wedged half-restarted state we are fixing. + The task re-polls without consuming a retry and runs once the drain + completes.""" + if task.function_name != JobSchedule.FN_NODE_RESTART: + return True + return not tasks_controller.is_auto_restart_paused(db.get_cluster_by_id(cluster.get_id())) + + +def _serialize(task, cluster): + if task.function_name != JobSchedule.FN_NODE_RESTART or not task.node_id: + return True + try: + return not _parallel_restart_allowed(db.get_storage_node_by_id(task.node_id)) + except KeyError: + return True + + def _get_node_unavailable_devices_count(node_id): node = db.get_storage_node_by_id(node_id) devices = [] @@ -46,62 +89,6 @@ def _validate_no_task_node_restart(cluster_id, node_id): return True -def _task_finish(task, result): - """Terminal task write via atomic CAS: set DONE + result on the FRESH row. - - A plain ``task.write_to_db()`` of the runner's in-memory copy writes the - whole stale object: it erased a concurrent cancellation and wiped the - owner lease (2026-07-29 double restart). This never overwrites an - already-DONE task (a concurrent cancellation's result wins) and preserves - ``owner`` / ``canceled`` as found. - - Returns True if this call performed the transition, False if the task was - already finished (or gone) — callers gating side effects (give-up - OFFLINE flip, re-queue) on having won the transition must check it. - """ - now = str(datetime.datetime.now(datetime.timezone.utc)) - wrote = {"done": False} - - def _mutate(t): - if t.status == JobSchedule.STATUS_DONE: - return False - t.function_result = result - t.status = JobSchedule.STATUS_DONE - t.updated_at = now - wrote["done"] = True - return True - - if db.atomic_update(task, _mutate) is None: - return False - return wrote["done"] - - -def _task_update(task, mutate): - """Non-terminal task write via atomic CAS. Applies ``mutate(t)`` to the - FRESH row, so it can neither resurrect a concurrently canceled/finished - task nor clobber concurrently-written fields (owner lease, cancellation) - the way a full-object ``write_to_db()`` of a stale copy does. - - Returns the fresh post-write task object, or None when the task is - done/canceled/gone — the caller must stop driving the task in that case. - ``mutate`` may be replayed on transaction conflict; it must only mutate - the object passed to it (no I/O, no other writes). - """ - now = str(datetime.datetime.now(datetime.timezone.utc)) - - def _mutate(t): - if t.status == JobSchedule.STATUS_DONE or t.canceled: - return False - mutate(t) - t.updated_at = now - return True - - fresh = db.atomic_update(task, _mutate) - if fresh is None or fresh.status == JobSchedule.STATUS_DONE or fresh.canceled: - return None - return fresh - - def _ensure_spdk_killed(node): """Best-effort kill of the SPDK process on the node before we mark it OFFLINE. Without this, flipping the status to OFFLINE while SPDK is still @@ -241,53 +228,24 @@ def task_runner(task): def task_runner_device(task): device = _get_device(task) - if 0 <= task.max_retry <= task.retry: - task.function_result = "max retry reached" - task.status = JobSchedule.STATUS_DONE - task.write_to_db(db.kv_store) - device_controller.device_set_unavailable(device.get_id()) - device_controller.device_set_retries_exhausted(device.get_id(), True) - return True - if not _validate_no_task_node_restart(task.cluster_id, task.node_id): - task.function_result = "canceled: node restart found" - task.status = JobSchedule.STATUS_DONE - task.write_to_db(db.kv_store) + # The node-level restart supersedes this device-level one. device_controller.device_set_unavailable(device.get_id()) - return True - - if task.canceled: - task.function_result = "canceled" - task.status = JobSchedule.STATUS_DONE - task.write_to_db(db.kv_store) - device_controller.device_set_retries_exhausted(device.get_id(), True) - return True + raise TaskAbort("canceled: node restart found") node = db.get_storage_node_by_id(task.node_id) if node.status != StorageNode.STATUS_ONLINE: logger.error(f"Node is not online: {node.get_id()}, retry") - task.function_result = "Node is offline" - task.retry += 1 - task.write_to_db(db.kv_store) - return False + raise TaskRetry("Node is offline") if device.status == NVMeDevice.STATUS_ONLINE and device.io_error is False: logger.info(f"Device is online: {device.get_id()}") task.function_result = "Device is online" - task.status = JobSchedule.STATUS_DONE - task.write_to_db(db.kv_store) - return True + return if device.status in [NVMeDevice.STATUS_REMOVED, NVMeDevice.STATUS_FAILED]: logger.info(f"Device is not unavailable: {device.get_id()}, {device.status} , stopping task") - task.function_result = f"stopped because dev is {device.status}" - task.status = JobSchedule.STATUS_DONE - task.write_to_db(db.kv_store) - return True - - if task.status != JobSchedule.STATUS_RUNNING: - task.status = JobSchedule.STATUS_RUNNING - task.write_to_db(db.kv_store) + raise TaskAbort(f"stopped because dev is {device.status}") # set device online for the first 3 retries if task.retry < 3: @@ -301,58 +259,81 @@ def task_runner_device(task): # check device status time.sleep(5) device = _get_device(task) - if device.status == NVMeDevice.STATUS_ONLINE and device.io_error is False: - logger.info(f"Device is online: {device.get_id()}") - task.function_result = "done" - task.status = JobSchedule.STATUS_DONE - task.write_to_db(db.kv_store) + if device.status != NVMeDevice.STATUS_ONLINE or device.io_error is not False: + raise TaskRetry(f"Device is {device.status}, retry") - tasks_controller.add_device_mig_task_for_node(task.node_id) + logger.info(f"Device is online: {device.get_id()}") + task.function_result = "done" + tasks_controller.add_device_mig_task_for_node(task.node_id) - return True - task.retry += 1 - task.write_to_db(db.kv_store) - return False +def _give_up_on_device(task): + """The retry ceiling has terminated a device-restart task.""" + device = _get_device(task) + if device is None: + return + device_controller.device_set_unavailable(device.get_id()) + device_controller.device_set_retries_exhausted(device.get_id(), True) + + +def _abandon_task(task): + """Driver on_finish: the terminal paths a handler never reaches. + + Only two need anything. The retry ceiling gives up on the target, which for + a node means parking it OFFLINE and re-queueing, and for a device means + marking it unavailable and out of retries. A cancellation stops a device + task retrying forever. Everything else — success, the aborts the handler + raises itself — leaves the target alone. + """ + ceiling_reached = 0 <= task.max_retry <= task.retry + + if task.function_name == JobSchedule.FN_NODE_RESTART: + if ceiling_reached and not task.canceled: + _give_up_on_node(task) + return + + if task.canceled: + device = _get_device(task) + if device is not None: + device_controller.device_set_retries_exhausted(device.get_id(), True) + elif ceiling_reached: + _give_up_on_device(task) + + +def _give_up_on_node(task): + """The retry ceiling has terminated a node-restart task. Reached through + the driver's on_finish, since the handler never sees that path.""" + # restart_cleanup: this task ran try_set_node_restarting earlier + # and is the lock owner; tagging unblocks the RESTARTING-lock + # guard so the giving-up flip lands. + storage_node_ops.set_node_status( + task.node_id, StorageNode.STATUS_OFFLINE, caused_by="restart_cleanup") + # Re-queue a fresh auto-restart task so the node does not get + # stranded in OFFLINE forever. Without this, the legitimate + # auto-restart trigger (set_node_offline) won't fire either — + # it skips when status is already OFFLINE — so the only path + # back is operator intervention. Hours-of-backoff exhaustion + # almost always means a long peer-side recovery is in flight; + # once it clears, the new task can succeed. + try: + node_obj = db.get_storage_node_by_id(task.node_id) + tasks_controller.add_node_to_auto_restart(node_obj) + except KeyError: + logger.debug( + f"Node {task.node_id} no longer exists, skipping auto-restart re-queue") + except Exception as exc: + logger.error(f"Failed to re-queue auto-restart for {task.node_id}: {exc}") def task_runner_node(task): try: node = db.get_storage_node_by_id(task.node_id) except KeyError: - _task_finish(task, "node not found") - return True - - if 0 <= task.max_retry <= task.retry: - if not _task_finish(task, "max retry reached"): - # Concurrently finished/canceled — someone else owns the outcome; - # the give-up side effects (OFFLINE flip + re-queue) must not run. - return True - # restart_cleanup: this task ran try_set_node_restarting earlier - # and is the lock owner; tagging unblocks the RESTARTING-lock - # guard so the giving-up flip lands. - storage_node_ops.set_node_status( - task.node_id, StorageNode.STATUS_OFFLINE, caused_by="restart_cleanup") - # Re-queue a fresh auto-restart task so the node does not get - # stranded in OFFLINE forever. Without this, the legitimate - # auto-restart trigger (set_node_offline) won't fire either — - # it skips when status is already OFFLINE — so the only path - # back is operator intervention. Hours-of-backoff exhaustion - # almost always means a long peer-side recovery is in flight; - # once it clears, the new task can succeed. - try: - node_obj = db.get_storage_node_by_id(task.node_id) - tasks_controller.add_node_to_auto_restart(node_obj) - except KeyError: - pass - except Exception as exc: - logger.error(f"Failed to re-queue auto-restart for {task.node_id}: {exc}") - return True + raise TaskAbort("node not found") if node.status in [StorageNode.STATUS_REMOVED, StorageNode.STATUS_SCHEDULABLE]: logger.info(f"Node is {node.status}, stopping task") - _task_finish(task, f"Node is {node.status}, stopping") - return True + raise TaskAbort(f"Node is {node.status}, stopping") # DOWN used to short-circuit here too. After removing the monitor's # set_node_online (which previously did DOWN -> ONLINE on health-check # pass), DOWN must be handled by this runner: shutdown + restart drives @@ -384,34 +365,28 @@ def task_runner_node(task): # a still-False health_check from auxiliary checks. if node.status == StorageNode.STATUS_ONLINE: logger.info(f"Node is online: {node.get_id()}") - _task_finish(task, "Node is online") - return True - - if task.canceled: - _task_finish(task, "canceled") - return True + task.function_result = "Node is online" + return - if task.status != JobSchedule.STATUS_RUNNING: - if node.status == StorageNode.STATUS_RESTARTING: - logger.info("Node is restarting, stopping task") - _task_finish(task, "Node is restarting") - return True - updated = _task_update( - task, lambda t: setattr(t, "status", JobSchedule.STATUS_RUNNING)) - if updated is None: - logger.info(f"Task {task.uuid} canceled/finished concurrently; stopping") - return True - task = updated + # A restart already in flight makes this task redundant — unless it is our + # own from an earlier attempt. This used to be inferred from the task still + # being NEW/SUSPENDED, which no longer distinguishes anything now that the + # driver moves the task to RUNNING before calling; the marker below records + # the fact directly, and also covers an attempt that issued the restart and + # then died, whose RESTARTING is ours to finish rather than defer to. + if (node.status == StorageNode.STATUS_RESTARTING + and not task.function_params.get("restart_issued")): + logger.info("Node is restarting, stopping task") + raise TaskAbort("Node is restarting") # Peer-restart mutual-exclusion pre-check: if any peer is RESTARTING # or IN_SHUTDOWN we cannot proceed (try_set_node_restarting in the # restart impl uses an FDB-tx with the same predicate and would fail # acquisition). This is purely transient — burning a retry on a lock - # we know we can't acquire just collapses the backoff budget. Return - # False without incrementing task.retry; the runner's outer loop - # will sleep with exponential backoff and re-call us. Once the peer - # finishes its transition, this check passes and we proceed with a - # fresh budget. + # we know we can't acquire just collapses the backoff budget, so it + # defers instead: no retry consumed, re-polled on the next short pass. + # Once the peer finishes its transition, this check passes and we + # proceed with a fresh budget. # # Skipped only for a SUSPENDED **and drained** cluster: recovery restarts # run in parallel then (see the dispatch loop below) so peers in @@ -419,10 +394,7 @@ def task_runner_node(task): # restart_storage_node is relaxed the same way (allow_concurrent_peers). # An operator-caused suspension never drains — its survivors still serve # IO — so it keeps the full pre-check. - cluster_obj = db.get_cluster_by_id(node.cluster_id) - if not (cluster_obj.status == Cluster.STATUS_SUSPENDED - and cluster_obj.suspend_drain_complete) \ - and not storage_node_ops.fd_dead_recovery_allowed(db, node): + if not _parallel_restart_allowed(node): # Strict one-restart-at-a-time outside the two sanctioned cases: # drained suspension, and a fully-dead failure domain # (fd_dead_recovery_allowed — no domain member ONLINE, so parallel @@ -435,17 +407,8 @@ def task_runner_node(task): continue if peer.status in (StorageNode.STATUS_RESTARTING, StorageNode.STATUS_IN_SHUTDOWN): - msg = (f"Peer {peer.get_id()[:8]} is {peer.status}; " - f"deferring (no retry consumed)") - logger.info(msg) - # Atomic CAS, NOT write_to_db of the stale in-memory copy: - # this exact write un-canceled a task that set_node_status - # (ONLINE) had canceled seconds earlier and erased its owner - # lease, resurrecting the restart (2026-07-29 double restart). - if _task_update( - task, lambda t, m=msg: setattr(t, "function_result", m)) is None: - return True - return False + raise TaskDefer(f"Peer {peer.get_id()[:8]} is {peer.status}; " + f"deferring (no retry consumed)") # is node reachable? ping_check = health_controller._check_node_ping(node.mgmt_ip) @@ -465,15 +428,8 @@ def task_runner_node(task): if data_ping_check is True: node_data_nic_ping_check = True if not ping_check or not node_api_check or not node_data_nic_ping_check: - # node is unreachable, retry logger.info(f"Node is not reachable: {task.node_id}, retry") - - def _unreachable(t): - t.function_result = "Node is unreachable, retry" - t.retry += 1 - if _task_update(task, _unreachable) is None: - return True - return False + raise TaskRetry("Node is unreachable, retry") # Last-line defense before the destructive shutdown/restart sequence: # everything above ran against reads taken seconds ago (the reachability @@ -481,17 +437,13 @@ def _unreachable(t): # (set_node_status(ONLINE) -> cancel_pending_node_restart_tasks) must stop # this entry HERE — in the 2026-07-29 double restart the second entry never # re-checked and force-shut a node that was back up and serving. - try: - task = db.get_task_by_id(task.uuid) - except KeyError: - return True - if task.canceled or task.status == JobSchedule.STATUS_DONE: + fresh = db.get_task_by_id(task.uuid) + if fresh is None or fresh.canceled or fresh.status == JobSchedule.STATUS_DONE: logger.info( f"Task {task.uuid} was canceled/finished concurrently; " f"stopping before shutdown") - if task.status != JobSchedule.STATUS_DONE: - _task_finish(task, "canceled") - return True + raise TaskAbort("canceled") + task = fresh # Cross-actor claim check on a fresh node read: a live driver (e.g. a # manual `sn restart`) mid-transition on this node holds the per-node @@ -505,18 +457,12 @@ def _unreachable(t): try: node = db.get_storage_node_by_id(task.node_id) except KeyError: - _task_finish(task, "node not found") - return True + raise TaskAbort("node not found") if node.status in (StorageNode.STATUS_RESTARTING, StorageNode.STATUS_IN_SHUTDOWN): claim_holder = db_controller.restart_claim_active(node) if claim_holder: - msg = (f"Node restart claim held by {claim_holder}; " - f"deferring (no retry consumed)") - logger.info(msg) - if _task_update( - task, lambda t, m=msg: setattr(t, "function_result", m)) is None: - return True - return False + raise TaskDefer(f"Node restart claim held by {claim_holder}; " + f"deferring (no retry consumed)") # Cleanup shutdown before the restart — but only when there is something # to clean: a node that is already OFFLINE had SPDK confirmed gone (that @@ -544,23 +490,25 @@ def _unreachable(t): if ret: logger.info("Node shutdown succeeded") shutdown_succeeded = True - - def _mark_cleanup(t): - t.function_params = dict(t.function_params) - t.function_params["cleanup_shutdown_done"] = True - updated = _task_update(task, _mark_cleanup) + updated = checkpoint(task, cleanup_shutdown_done=True) if updated is None: # Canceled under us right after the shutdown; do not # drive the restart of a canceled task. The monitor's # offline re-queue scan picks the node up again. - return True + raise TaskAbort("canceled during cleanup shutdown") task = updated else: logger.error("Node shutdown returned False; will retry after reset") time.sleep(3) + except (TaskAbort, TaskDefer, TaskRetry): + raise except Exception as e: logger.error(e) - return False + # Preserved as a defer, not a failure: this branch never + # consumed a retry, and restart's give-up has side effects + # (OFFLINE flip + re-queue) that a changed verdict would start + # triggering where it previously could not. + raise TaskDefer(f"cleanup shutdown raised: {e}") else: logger.info( f"Skipping cleanup shutdown for {node.get_id()}: " @@ -571,9 +519,7 @@ def _mark_cleanup(t): # of a half-shutdown node produced the in_restart hang we're guarding # against. Let the outer retry reattempt the whole cycle. if not shutdown_succeeded: - if _task_update(task, lambda t: setattr(t, "retry", t.retry + 1)) is None: - return True - return False + raise TaskRetry("Node shutdown did not succeed") try: # resetting node @@ -586,12 +532,21 @@ def _mark_cleanup(t): # never recognized as "ours" — masked only because this call # uses force=True, which proceeds past the guard regardless and # just logged a spurious "Restart task found" error every time. + # Recorded before the call: a restart that starts and then loses + # this process still owns the node's RESTARTING state, and the next + # attempt must recognise it as ours rather than stopping for it. + updated = checkpoint(task, restart_issued=True) + if updated is None: + raise TaskAbort("canceled before restart") + task = updated ret = storage_node_ops.restart_storage_node(node.get_id(), force=True, current_restart_task_id=task.uuid) if ret: logger.info("Node restart succeeded") + except (TaskAbort, TaskDefer, TaskRetry): + raise except Exception as e: logger.error(e) - return False + raise TaskDefer(f"restart raised: {e}") time.sleep(3) node = db.get_storage_node_by_id(task.node_id) @@ -625,12 +580,10 @@ def _mark_cleanup(t): # False at the moment we re-read the DB. if node.status == StorageNode.STATUS_ONLINE: logger.info(f"Node is online: {node.get_id()}") - _task_finish(task, "done") - return True + task.function_result = "done" + return - if _task_update(task, lambda t: setattr(t, "retry", t.retry + 1)) is None: - return True - return False + raise TaskRetry("Node did not come back online") finally: # On any non-success exit from the shutdown/restart sequence, make sure # we don't leave the node pinned in STATUS_IN_SHUTDOWN or @@ -646,21 +599,11 @@ def _mark_cleanup(t): logger.error(f"Post-task status reset check failed: {exc}") -# Per-task restart scheduling (in-memory; this runner is a single long-lived -# process). Maps task uuid -> epoch time when the task is next eligible to run. -# This lets the runner round-robin all restart tasks instead of pinning the -# thread on one task's blocking retry loop: a task that is waiting — deferred on -# a concurrent peer restart, or in failure backoff — no longer blocks the other -# pending restart tasks behind it (incident 2026-06-25: a single task deferring -# on a peer that was briefly in_restart sat in a growing backoff and starved a -# second node's brand-new restart task indefinitely). -_restart_next_attempt: dict = {} - # A genuine restart FAILURE first retries at a steady 1-minute cadence for a few # attempts (so a node that just needs a moment to come back recovers quickly), -# then falls back to the existing exponential backoff capped at +# then falls back to exponential backoff capped at # RESTART_TASK_EXEC_INTERVAL_MAX_SEC. A DEFER (peer-restart mutual exclusion) is -# NOT a failure and does not back off at all — see the loop below. +# NOT a failure and does not back off at all — the driver re-polls it next pass. RESTART_LEAD_IN_RETRIES = 3 RESTART_LEAD_IN_INTERVAL_SEC = 60 @@ -668,8 +611,8 @@ def _mark_cleanup(t): def _restart_backoff_seconds(retry): """Delay before the next attempt of a FAILED restart (one that consumed a retry). First RESTART_LEAD_IN_RETRIES attempts use a constant 1-minute - cadence; after that the existing exponential backoff applies, continuing - upward from the lead-in interval and capped at the configured maximum.""" + cadence; after that exponential backoff applies, continuing upward from the + lead-in interval and capped at the configured maximum.""" if retry <= RESTART_LEAD_IN_RETRIES: return RESTART_LEAD_IN_INTERVAL_SEC exp = RESTART_LEAD_IN_INTERVAL_SEC * (2 ** (retry - RESTART_LEAD_IN_RETRIES)) @@ -752,182 +695,37 @@ def _watchdog_orphaned_transitional_nodes(cluster_id): logger.info(f"Queued auto-restart for recovered node {node_id}") -# Parallel restart execution for SUSPENDED clusters: during full-cluster -# recovery every node is offline and no client IO flows, so node restarts -# cannot violate FTT and are fanned out on this pool (~70 s each; strictly -# sequential recovery of a 32-node cluster took ~38 min, 2026-07-08). The -# per-primary consistency of the cross-node connect section is preserved by -# storage_node_ops._remote_connect_gate, and the peer-exclusion guards -# (task_runner_node pre-check + try_set_node_restarting) are relaxed only -# while the cluster is SUSPENDED. Online clusters never dispatch here. -_restart_pool = ThreadPoolExecutor( - max_workers=constants.NODE_RESTART_MAX_PARALLEL_SUSPENDED, - thread_name_prefix="node-restart") -_restart_inflight: dict = {} # task uuid -> Future -# node_id -> Future. Parallel dispatch MUST also be exclusive per NODE, not -# only per task: multiple node_restart tasks can be queued for the same node -# (escalation + requeue paths), and keying inflight by task uuid alone let -# them run concurrently — each kill-and-restarting the same SPDK out from -# under the other, flipping the node offline/in_restart in a loop (observed -# 2026-07-10 mass-reboot recovery: 79 concurrent same-node dispatches, nodes -# stuck bouncing for 10+ minutes). -_node_inflight: dict = {} - - -def _process_restart_task(task_uuid): - """Claim and drive one restart task, including the per-task backoff - bookkeeping. Runs inline (serialized) normally, or on the - suspended-cluster parallel pool. Never raises: a crash in one task must - not kill recovery of every other node.""" - try: - # Re-read (it may have been canceled / changed concurrently). - task = db.get_task_by_id(task_uuid) - if task.status == JobSchedule.STATUS_DONE: - _restart_next_attempt.pop(task_uuid, None) - return - # Lease gate: do not drive a task another live runner host - # already owns (prevents a second replica issuing a - # concurrent shutdown/restart). - if not tasks_controller.claim_task(task): - logger.info(f"Restart task {task_uuid} owned by another runner host; skipping") - return - retry_before = task.retry - # Device restarts (and parts of node restarts outside the - # restart_storage_node wrapper) block without task writes; heartbeat - # the lease so it never goes stale mid-execution and gets stolen by - # another runner host. - with tasks_controller.task_lease_heartbeat(task): - res = task_runner(task) - task = db.get_task_by_id(task_uuid) - if res or task.status == JobSchedule.STATUS_DONE: - _restart_next_attempt.pop(task_uuid, None) - elif task.retry > retry_before: - # Genuine failure (retry consumed): 1-min lead-in, then - # exponential backoff. - _restart_next_attempt[task_uuid] = ( - time.time() + _restart_backoff_seconds(task.retry)) - else: - # Defer (peer-restart mutual exclusion; retry NOT - # consumed): not a failure — do not back off. Re-poll on - # the next short pass so this picks up immediately once - # the blocking restart finishes. - _restart_next_attempt[task_uuid] = ( - time.time() + constants.RESTART_TASK_EXEC_INTERVAL_SEC) - except Exception as e: - logger.error(f"Restart task {task_uuid} processing crashed: {e}") - logger.exception(e) - try: - retry = db.get_task_by_id(task_uuid).retry - except Exception: - retry = 0 - _restart_next_attempt[task_uuid] = ( - time.time() + _restart_backoff_seconds(retry)) +SPEC = RunnerSpec( + name="tasks-runner-restart", + function_names=[JobSchedule.FN_DEV_RESTART, JobSchedule.FN_NODE_RESTART], + handler=task_runner, + on_finish=_abandon_task, + on_cycle=lambda cluster: _watchdog_orphaned_transitional_nodes(cluster.get_id()), + is_eligible=_is_eligible, + interval=constants.TASK_EXEC_INTERVAL_SEC, + # Parallel restart execution for SUSPENDED clusters: during full-cluster + # recovery every node is offline and no client IO flows, so node restarts + # cannot violate FTT and are fanned out (~70 s each; strictly sequential + # recovery of a 32-node cluster took ~38 min, 2026-07-08). The per-primary + # consistency of the cross-node connect section is preserved by + # storage_node_ops._remote_connect_gate, and the peer-exclusion guards + # (the pre-check in task_runner_node + try_set_node_restarting) are relaxed + # under exactly the same condition. Online clusters stay sequential. + concurrency=constants.NODE_RESTART_MAX_PARALLEL_SUSPENDED, + serialize=_serialize, + # Never two restart tasks for the same node at once: multiple node_restart + # tasks can be queued for one node (escalation + requeue paths), and + # excluding by task alone let them run concurrently — each kill-and- + # restarting the same SPDK out from under the other, flipping the node + # offline/in_restart in a loop (2026-07-10 mass-reboot recovery: 79 + # concurrent same-node dispatches, nodes bouncing for 10+ minutes). + exclusion_key=lambda task: task.node_id or None, + backoff=_restart_backoff_seconds, +) def main(): - logger.info("Starting Tasks runner...") - while True: - clusters = db.get_clusters() - if not clusters: - logger.error("No clusters found!") - else: - for cl in clusters: - tasks = db.get_job_tasks(cl.get_id(), reverse=False) - for task in tasks: - if task.function_name not in [JobSchedule.FN_DEV_RESTART, JobSchedule.FN_NODE_RESTART]: - continue - if task.status == JobSchedule.STATUS_DONE: - _restart_next_attempt.pop(task.uuid, None) - _restart_inflight.pop(task.uuid, None) - continue - # Round-robin: skip a task that is not yet due so a waiting task - # (deferred on a concurrent peer restart, or in failure backoff) - # does NOT block the other pending restart tasks behind it. The - # outer loop revisits every task each pass (TASK_EXEC_INTERVAL_SEC). - if time.time() < _restart_next_attempt.get(task.uuid, 0): - continue - # Suspend recovery: while a SUSPENDED cluster is still being - # drained to all-offline, pause node restarts. Executing one now - # would fight the auto-shutdown and re-create the wedged - # half-restarted state we are fixing. Re-poll soon without - # consuming a retry; the task runs once the drain completes - # (suspend_drain_complete). - dispatch_parallel = False - if task.function_name == JobSchedule.FN_NODE_RESTART: - cl_fresh = db.get_cluster_by_id(cl.get_id()) - if tasks_controller.is_auto_restart_paused(cl_fresh): - logger.info( - "Cluster %s suspended and draining; deferring " - "node-restart task %s", cl.get_id(), task.uuid) - _restart_next_attempt[task.uuid] = ( - time.time() + constants.RESTART_TASK_EXEC_INTERVAL_SEC) - continue - # SUSPENDED and drained: full-cluster recovery — fan node - # restarts out on the pool (see _restart_pool). Online - # clusters stay strictly sequential. suspend_drain_complete - # is required: it certifies every (non operator-stopped) - # node went OFFLINE, i.e. no client IO — an operator-caused - # suspension never drains, its survivors are still serving, - # and its restarts must stay sequential with full guards. - # Parallel dispatch in the two sanctioned cases only: - # drained suspension (full-cluster recovery), and a - # fully-dead failure domain (fd_dead_recovery_allowed: - # no domain member ONLINE, no cross-domain restart in - # flight — recovery of a rebooted domain fans out - # instead of 16 x single-restart serially). The former - # relaxation that fanned out while the domain was - # still SERVING (2026-07-16 violation) stays removed. - dispatch_parallel = (cl_fresh.status == Cluster.STATUS_SUSPENDED - and cl_fresh.suspend_drain_complete) - if not dispatch_parallel and task.node_id: - try: - dispatch_parallel = storage_node_ops.fd_dead_recovery_allowed( - db, db.get_storage_node_by_id(task.node_id)) - except KeyError: - pass - - # Single dispatch path: EVERY execution goes through the - # pool with _restart_inflight/_node_inflight as the - # mutual-exclusion authority. The former split — a parallel - # branch that consulted the inflight maps and an inline - # branch that consulted neither — let a dispatch-mode flip - # mid-restart (fd_dead_recovery_allowed going false as the - # first peers of the domain came back ONLINE) re-enter the - # SAME task that was still running on the pool, which then - # force-shut the already-recovered node (2026-07-29 double - # restart). Serialized mode submits identically and just - # waits for the future, so the bookkeeping is the same in - # both modes and mode flips are harmless in both directions. - inflight = _restart_inflight.get(task.uuid) - if inflight is not None and not inflight.done(): - continue - # Per-node exclusion: never run two restart tasks for the - # same node concurrently (see _node_inflight above). The - # duplicate task re-polls next pass; by then the winner - # has usually completed and marked it obsolete. - node_inflight = _node_inflight.get(task.node_id) - if node_inflight is not None and not node_inflight.done(): - _restart_next_attempt[task.uuid] = ( - time.time() + constants.RESTART_TASK_EXEC_INTERVAL_SEC) - continue - fut = _restart_pool.submit(_process_restart_task, task.uuid) - _restart_inflight[task.uuid] = fut - if task.node_id: - _node_inflight[task.node_id] = fut - if not dispatch_parallel: - # Inline (serialized) execution: wait for this task - # before dispatching the next. _process_restart_task - # never raises, so a crash in one task cannot escape to - # the outer `while True` and kill recovery of every - # other node. - fut.result() - - try: - _watchdog_orphaned_transitional_nodes(cl.get_id()) - except Exception as e: - logger.error(f"Orphaned-node watchdog failed for cluster {cl.get_id()}: {e}") - - time.sleep(constants.TASK_EXEC_INTERVAL_SEC) + serve(SPEC) if __name__ == "__main__": diff --git a/tests/integration/test_tasks_runner_restart_shortcircuit.py b/tests/integration/test_tasks_runner_restart_shortcircuit.py index 6e2bdba861..4134743dcf 100644 --- a/tests/integration/test_tasks_runner_restart_shortcircuit.py +++ b/tests/integration/test_tasks_runner_restart_shortcircuit.py @@ -39,6 +39,7 @@ from simplyblock_core.models.job_schedule import JobSchedule from simplyblock_core.models.storage_node import StorageNode from simplyblock_core.models.nvme_device import NVMeDevice +from simplyblock_core.services.task_runner_base import TaskAbort, TaskRetry _RUNNER_PATH = os.path.join( @@ -126,9 +127,7 @@ def test_online_and_healthy_with_no_devices_skips_restart(self): node = _mk_node(status=StorageNode.STATUS_ONLINE, health_check=True, nvme_devices=[]) with patch.object(mod, "db", _mk_db(node)): - ret = mod.task_runner_node(task) - self.assertTrue(ret) - self.assertEqual(task.status, JobSchedule.STATUS_DONE) + mod.task_runner_node(task) self.assertIn("online", task.function_result.lower()) def test_online_and_healthy_with_unavailable_devices_still_skips(self): @@ -143,9 +142,8 @@ def test_online_and_healthy_with_unavailable_devices_still_skips(self): node = _mk_node(status=StorageNode.STATUS_ONLINE, health_check=True, nvme_devices=[bad_dev]) with patch.object(mod, "db", _mk_db(node)): - ret = mod.task_runner_node(task) - self.assertTrue(ret) - self.assertEqual(task.status, JobSchedule.STATUS_DONE) + mod.task_runner_node(task) + self.assertIn("online", task.function_result.lower()) def test_online_but_unhealthy_still_skips_restart(self): """Critical regression: an ONLINE node with health_check=False @@ -158,9 +156,7 @@ def test_online_but_unhealthy_still_skips_restart(self): node = _mk_node(status=StorageNode.STATUS_ONLINE, health_check=False, nvme_devices=[]) with patch.object(mod, "db", _mk_db(node)): - ret = mod.task_runner_node(task) - self.assertTrue(ret) - self.assertEqual(task.status, JobSchedule.STATUS_DONE) + mod.task_runner_node(task) self.assertIn("online", task.function_result.lower()) @@ -180,8 +176,10 @@ def test_offline_does_not_short_circuit(self): mock_health._check_node_ping.return_value = False mock_health._check_node_api.return_value = False mock_health._check_ping_from_node.return_value = False - _ = mod.task_runner_node(task) - self.assertNotEqual(task.status, JobSchedule.STATUS_DONE) + # Not a short-circuit: it falls through to the reachability + # checks and asks the driver for another attempt. + with self.assertRaises(TaskRetry): + mod.task_runner_node(task) class TestTerminalStatusesStillDoneImmediately(unittest.TestCase): @@ -201,9 +199,10 @@ def test_removed_short_circuits_without_restart(self): task = _mk_task() node = _mk_node(status=StorageNode.STATUS_REMOVED) with patch.object(mod, "db", _mk_db(node)): - ret = mod.task_runner_node(task) - self.assertTrue(ret) - self.assertEqual(task.status, JobSchedule.STATUS_DONE) + # TaskAbort is the terminal, non-retryable stop the driver turns + # into DONE — reached before any shutdown/restart call. + with self.assertRaises(TaskAbort): + mod.task_runner_node(task) if __name__ == "__main__": diff --git a/tests/unit/tasks/test_max_retry_semantics.py b/tests/unit/tasks/test_max_retry_semantics.py index 761e8b21e9..a7e5287981 100644 --- a/tests/unit/tasks/test_max_retry_semantics.py +++ b/tests/unit/tasks/test_max_retry_semantics.py @@ -105,64 +105,94 @@ def test_node_add_exception_is_handled_like_a_failed_add(monkeypatch): # -------------------------------------------------------------------------- -# tasks_runner_restart.task_runner_node +# tasks_runner_restart give-up side effects +# +# The ceiling itself is the driver's (test_task_runner_base); what the runner +# still owns is what to do about the target when the ceiling terminates a task +# — a path its handler never sees, so it hangs off the driver's on_finish. # -------------------------------------------------------------------------- -def test_restart_node_max_retry_finishes_and_marks_offline(monkeypatch): - """retry >= max_retry finishes the task, flips the node OFFLINE via the - restart_cleanup path, and re-queues a fresh auto-restart.""" - node = MagicMock() - node.status = StorageNode.STATUS_ONLINE +def _restart_task(function_name, retry, max_retry, canceled=False): + task = _task(function_name, retry, max_retry, node_id="node-1") + task.canceled = canceled + return task + + +def test_restart_node_give_up_marks_offline_and_requeues(monkeypatch): + """Exhausting the retries parks the node OFFLINE and queues a fresh + auto-restart, so it is not stranded until an operator intervenes.""" fake_db = MagicMock() - fake_db.get_storage_node_by_id.return_value = node - # Mirror DBController.atomic_update's contract: apply the mutator to the - # (fresh) object and return it. The restart runner's task writes go - # through this instead of write_to_db (stale-copy lost-update fix). - fake_db.atomic_update.side_effect = lambda obj, fn: (fn(obj), obj)[1] monkeypatch.setattr(restart_runner, "db", fake_db) - sops = MagicMock() tasks_ctrl = MagicMock() monkeypatch.setattr(restart_runner, "storage_node_ops", sops) monkeypatch.setattr(restart_runner, "tasks_controller", tasks_ctrl) - task = _task(JobSchedule.FN_NODE_RESTART, retry=5, max_retry=5, node_id="node-1") - res = restart_runner.task_runner_node(task) + restart_runner.SPEC.on_finish( + _restart_task(JobSchedule.FN_NODE_RESTART, retry=5, max_retry=5)) - assert res is True - assert task.status == JobSchedule.STATUS_DONE - assert "max retry" in task.function_result sops.set_node_status.assert_called_once() assert sops.set_node_status.call_args.args[1] == StorageNode.STATUS_OFFLINE tasks_ctrl.add_node_to_auto_restart.assert_called_once() -# -------------------------------------------------------------------------- -# tasks_runner_restart.task_runner_device -# -------------------------------------------------------------------------- +def test_restart_node_success_leaves_the_node_alone(monkeypatch): + """A task that finished below the ceiling succeeded — the node is online + and must not be flipped OFFLINE by the cleanup path.""" + monkeypatch.setattr(restart_runner, "db", MagicMock()) + sops = MagicMock() + tasks_ctrl = MagicMock() + monkeypatch.setattr(restart_runner, "storage_node_ops", sops) + monkeypatch.setattr(restart_runner, "tasks_controller", tasks_ctrl) + + restart_runner.SPEC.on_finish( + _restart_task(JobSchedule.FN_NODE_RESTART, retry=2, max_retry=5)) + + sops.set_node_status.assert_not_called() + tasks_ctrl.add_node_to_auto_restart.assert_not_called() -def test_restart_device_max_retry_finishes_and_exhausts(monkeypatch): - """retry >= TASK_EXEC_RETRY_COUNT finishes the task and marks the device - unavailable / retries-exhausted instead of restarting it again.""" + +def test_restart_node_unbounded_never_gives_up(monkeypatch): + monkeypatch.setattr(restart_runner, "db", MagicMock()) + sops = MagicMock() + monkeypatch.setattr(restart_runner, "storage_node_ops", sops) + monkeypatch.setattr(restart_runner, "tasks_controller", MagicMock()) + + restart_runner.SPEC.on_finish( + _restart_task(JobSchedule.FN_NODE_RESTART, retry=100, max_retry=-1)) + + sops.set_node_status.assert_not_called() + + +def test_restart_device_give_up_marks_unavailable_and_exhausted(monkeypatch): device = MagicMock() device.get_id.return_value = "dev-1" monkeypatch.setattr(restart_runner, "_get_device", lambda task: device) monkeypatch.setattr(restart_runner, "db", MagicMock()) - dc = MagicMock() monkeypatch.setattr(restart_runner, "device_controller", dc) - task = _task( + restart_runner.SPEC.on_finish(_restart_task( JobSchedule.FN_DEV_RESTART, retry=constants.TASK_EXEC_RETRY_COUNT, - max_retry=constants.TASK_EXEC_RETRY_COUNT, - node_id="node-1", - ) - res = restart_runner.task_runner_device(task) - - assert res is True - assert task.status == JobSchedule.STATUS_DONE - assert "max retry" in task.function_result + max_retry=constants.TASK_EXEC_RETRY_COUNT)) + dc.device_set_unavailable.assert_called_once_with("dev-1") dc.device_set_retries_exhausted.assert_called_once_with("dev-1", True) dc.restart_device.assert_not_called() + + +def test_restart_device_cancel_exhausts_retries(monkeypatch): + """A canceled device task must not be picked up and retried forever.""" + device = MagicMock() + device.get_id.return_value = "dev-1" + monkeypatch.setattr(restart_runner, "_get_device", lambda task: device) + monkeypatch.setattr(restart_runner, "db", MagicMock()) + dc = MagicMock() + monkeypatch.setattr(restart_runner, "device_controller", dc) + + restart_runner.SPEC.on_finish(_restart_task( + JobSchedule.FN_DEV_RESTART, retry=0, max_retry=5, canceled=True)) + + dc.device_set_retries_exhausted.assert_called_once_with("dev-1", True) + dc.device_set_unavailable.assert_not_called() diff --git a/tests/unit/tasks/test_retry_ceiling.py b/tests/unit/tasks/test_retry_ceiling.py index 50f0206df4..70a4b00bc2 100644 --- a/tests/unit/tasks/test_retry_ceiling.py +++ b/tests/unit/tasks/test_retry_ceiling.py @@ -192,29 +192,6 @@ def _wire_base(runner, monkeypatch, task): # tree, so a new retry-driven runner shows up as a failing case until a spec is # added here. -def _spec_restart(runner, monkeypatch): - task = _make_task(JobSchedule.FN_NODE_RESTART) - _db, _cluster, node = _wire_base(runner, monkeypatch, task) - # Node is offline and stays unreachable -> restart keeps failing, retry - # each poll. - node.status = StorageNode.STATUS_OFFLINE - monkeypatch.setattr(runner, "_restart_pool", _InlineExecutor()) - monkeypatch.setattr(runner, "_restart_next_attempt", {}) - monkeypatch.setattr(runner, "_restart_inflight", {}) - monkeypatch.setattr(runner, "_node_inflight", {}) - monkeypatch.setattr(runner.tasks_controller, "is_auto_restart_paused", - lambda *a, **k: False) - monkeypatch.setattr(runner.tasks_controller, "add_node_to_auto_restart", - MagicMock()) - monkeypatch.setattr(runner.storage_node_ops, "set_node_status", MagicMock()) - # Node never reachable -> the reachability check fails and retry advances. - monkeypatch.setattr(runner.health_controller, "_check_node_ping", - lambda *a, **k: False) - monkeypatch.setattr(runner.health_controller, "_check_node_api", - lambda *a, **k: False) - return task - - def _spec_batch_migration(runner, monkeypatch): task = _make_task(JobSchedule.FN_LVOL_BATCH_MIG, group_id="grp-1") db, _cluster, _ = _wire_base(runner, monkeypatch, task) @@ -267,7 +244,6 @@ def _get_node(node_id): # name -> spec for the runners driven through their real main() loop. _MAIN_DRIVEN_SPECS = { - "tasks_runner_restart.py": _spec_restart, "tasks_runner_batch_migration.py": _spec_batch_migration, } @@ -292,6 +268,7 @@ def _get_node(node_id): "tasks_runner_backup.py", "tasks_runner_cluster_expand.py", "tasks_runner_node_add.py", + "tasks_runner_restart.py", } # Runners that increment task.retry but are intentionally UNBOUNDED: the diff --git a/tests/unit/test_restart_claim.py b/tests/unit/test_restart_claim.py index 5a3481cd61..6597325498 100644 --- a/tests/unit/test_restart_claim.py +++ b/tests/unit/test_restart_claim.py @@ -22,8 +22,10 @@ - the wrapper's failure cleanup (_kill_spdk_until_dead + OFFLINE flip) runs only when THIS call actually holds the claim — a refused attempt must not destroy the rightful owner's in-flight restart; -- task_runner_node defers (no retry consumed) on a live foreign claim. +- task_runner_node raises TaskDefer (no retry consumed) on a live foreign + claim. """ +import contextlib import datetime import json from unittest.mock import MagicMock @@ -37,6 +39,8 @@ from simplyblock_core.models.storage_node import StorageNode import simplyblock_core.services.tasks_runner_restart as restart_runner +from simplyblock_core.services import task_runner_base +from simplyblock_core.services.task_runner_base import TaskAbort, TaskDefer, TaskRetry def _now(): @@ -312,7 +316,11 @@ def _runner_task(retry=0, max_retry=5, status=JobSchedule.STATUS_RUNNING): def test_runner_defers_without_retry_on_live_foreign_claim(monkeypatch): - claimed = _node(status=StorageNode.STATUS_RESTARTING, + # IN_SHUTDOWN, not RESTARTING: a node found RESTARTING without this task's + # own restart_issued marker is short-circuited earlier as redundant and + # never reaches the claim check. IN_SHUTDOWN is the shape the check was + # written for anyway — a CLI `sn restart` mid-shutdown holding the claim. + claimed = _node(status=StorageNode.STATUS_IN_SHUTDOWN, claim_owner="cli:100:aa", claim_age_sec=3) task = _runner_task() @@ -325,6 +333,9 @@ def test_runner_defers_without_retry_on_live_foreign_claim(monkeypatch): fake_db.get_storage_nodes_by_cluster_id.return_value = [] fake_db.atomic_update.side_effect = lambda obj, fn: (fn(obj), obj)[1] monkeypatch.setattr(restart_runner, "db", fake_db) + # The handler's checkpoint/CAS writes go through task_runner_base's own db + # handle, not the runner module's. + monkeypatch.setattr(task_runner_base, "db", fake_db) monkeypatch.setattr(JobSchedule, "write_to_db", MagicMock()) sops = MagicMock() @@ -336,17 +347,18 @@ def test_runner_defers_without_retry_on_live_foreign_claim(monkeypatch): hc._check_ping_from_node.return_value = True monkeypatch.setattr(restart_runner, "health_controller", hc) - res = restart_runner.task_runner_node(task) + # TaskDefer is the driver's no-retry-consumed signal; the driver, not the + # handler, records the reason and suspends the task. + with pytest.raises(TaskDefer, match="claim held by cli:100:aa"): + restart_runner.task_runner_node(task) - assert res is False # defer: outer loop backoff, retried later assert task.retry == 0 # no retry budget consumed - assert "claim held by cli:100:aa" in task.function_result sops.shutdown_storage_node.assert_not_called() sops.restart_storage_node.assert_not_called() def test_runner_proceeds_when_claim_stale(monkeypatch): - stale = _node(status=StorageNode.STATUS_RESTARTING, + stale = _node(status=StorageNode.STATUS_IN_SHUTDOWN, claim_owner="cli:100:aa", claim_age_sec=constants.RESTART_CLAIM_TTL_SEC + 30) task = _runner_task() @@ -360,6 +372,9 @@ def test_runner_proceeds_when_claim_stale(monkeypatch): fake_db.get_storage_nodes_by_cluster_id.return_value = [] fake_db.atomic_update.side_effect = lambda obj, fn: (fn(obj), obj)[1] monkeypatch.setattr(restart_runner, "db", fake_db) + # The handler's checkpoint/CAS writes go through task_runner_base's own db + # handle, not the runner module's. + monkeypatch.setattr(task_runner_base, "db", fake_db) monkeypatch.setattr(JobSchedule, "write_to_db", MagicMock()) sops = MagicMock() @@ -373,7 +388,12 @@ def test_runner_proceeds_when_claim_stale(monkeypatch): hc._check_ping_from_node.return_value = True monkeypatch.setattr(restart_runner, "health_controller", hc) - restart_runner.task_runner_node(task) + # Past the claim check the handler runs the full restart and ends on + # whatever the un-mocked completion check decides — irrelevant here, and + # suppressing it cannot mask a blocked claim: a blocked one raises + # TaskDefer before reaching the shutdown the assertion below requires. + with contextlib.suppress(TaskAbort, TaskDefer, TaskRetry): + restart_runner.task_runner_node(task) # The stale claim (dead driver) did not block the resume path: the # runner reached its cleanup shutdown step. From 8d6ce1025ae4f8c8d081b384464f92ada904c5c9 Mon Sep 17 00:00:00 2001 From: Max Schettler Date: Wed, 5 Aug 2026 20:54:43 +0200 Subject: [PATCH 21/25] Update handover document --- HANDOFF.md | 51 ++++++++++++++++++++++++++++++++++----------------- 1 file changed, 34 insertions(+), 17 deletions(-) diff --git a/HANDOFF.md b/HANDOFF.md index 30592ef0b5..65ccf391d4 100644 --- a/HANDOFF.md +++ b/HANDOFF.md @@ -21,29 +21,29 @@ ad4768462 Introduce generic task runner (B0) ``` -## Status: Phase A done, B0 done, B1–B3 done +## Status: Phase A done, B0 done, 8 of 13 runners migrated -Migrated onto the driver (7): `fdb_backup`, `jc_comp`, `replication_final`, -`sync_lvol_del`, `backup`, `cluster_expand`, `node_add`. +Migrated onto the driver: `fdb_backup`, `jc_comp`, `replication_final`, +`sync_lvol_del`, `backup`, `cluster_expand`, `node_add`, `restart`. -**Remaining (B4…B9), in order:** +**Remaining, in order:** -4. migration trio — `migration`, `new_dev_migration`, `failed_migration` +1. migration trio — `migration`, `new_dev_migration`, `failed_migration` (serial; `get_active_node_mig_task` / same-node-sibling gating becomes - `is_eligible`) -5. `port_allow` (large; its `is_eligible` omits the `IN_ACTIVATION` check — the + `is_eligible`). Note `tasks_controller.defer_if_cluster_expanding` writes + task state from inside these handlers — it becomes a `TaskDefer` (or an + `is_eligible` clause) and the helper goes away. +2. `port_allow` (large; its `is_eligible` omits the `IN_ACTIVATION` check — the documented opt-out — and keeps the recovery logic) -6. `restart` (concurrency + per-node `exclusion_key`; `is_auto_restart_paused` - and `fd_dead_recovery_allowed` become part of `is_eligible`) -7. `lvol_migration` (largest; migrate the loop/lease/retry shell only, leave the - snapshot-copy state machine intact) -8. `node_removal` (serial, already leased, unbounded) -9. `batch_migration` (migrate the loop/lease shell, leave group orchestration) +3. `node_removal` (serial, already leased, unbounded) -## What the driver grew during B1–B3 +**Deferred to a follow-up PR:** `lvol_migration` and `batch_migration` — under +active upstream rewrite (~17 commits in the last window, several still labelled +`TEMP:`). Migrating them now guarantees repeated conflicts for no benefit. -Beyond what the plan describes, `task_runner_base` gained two things the -migrations needed: +## What the driver grew during the migrations + +Beyond what the plan describes, `task_runner_base` gained: - **`function_result` is cleared before each handler attempt**, so a task that fails and later succeeds doesn't finish carrying the stale failure message. @@ -54,6 +54,22 @@ migrations needed: `sync_lvol_del` frees the primary's del-sync lock there, `backup` fails/ un-merges the backup resource there. Both are written to be no-ops when the handler already finished the resource. +- **All task writes are compare-and-set** (`_commit`), never full-object + `write_to_db`. This is not a refinement — the original driver reproduced the + lost update behind upstream's 2026-07-29 double-restart incident, and held the + stale copy for the whole handler duration. See the plan's "Upstream + reconciliation (2026-08)". +- **One dispatch path**: serialized execution submits to the pool and waits + rather than running inline, so the inflight registry is the single + mutual-exclusion authority. `RunnerSpec.serialize` is a per-task predicate, + because restart picks its mode from live cluster state. +- **`checkpoint(task, **params)`** — persist handler progress mid-handler, for a + destructive step that must not repeat after a crash. Doubles as the + cancellation probe before the next destructive step. +- **`RunnerSpec.on_cycle(cluster)`** — per-cluster upkeep attached to no task + (restart's orphaned-node watchdog). +- **`RunnerSpec.backoff(retry)`** — override the default curve where a runner + has a tuned one (restart's 1-minute lead-in). ## Gotchas @@ -75,7 +91,8 @@ migrations needed: ```bash tox run-parallel -e lint,types -tox run -e unit -- tests/unit/tasks/ tests/unit/test_lvol_sync_op_task.py +tox run -e unit -- tests/unit/tasks/ tests/unit/test_lvol_sync_op_task.py \ + tests/unit/test_task_cancellation.py tox run -e unit # full unit tier, ~45s, currently green ``` From 135391bfe54c3ea3992aea8fa62edc9dd74c624c Mon Sep 17 00:00:00 2001 From: Max Schettler Date: Thu, 6 Aug 2026 01:55:36 +0200 Subject: [PATCH 22/25] Migrate the device migration runners onto the task runner MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The three runners drive the same data-plane operation and differ only in what they migrate and when they may start, so the shared half moves to migration_task_common: the status poll, the start-and-record step, the cluster/expansion gate, the settle wait and the recovery gate. utils.handle_task_result and tasks_controller.defer_task_for_expansion were the last two places outside a runner that wrote task state; both are gone. The vocabulary gained TaskProgress for this family. An in-progress poll cannot suspend the task: get_active_node_mig_task keys the family's mutual exclusion on a sibling being RUNNING, so a suspended migration would let a second one start on the same node. Three things needed re-expressing rather than moving: - "has this task started its migration" was read off task.status being RUNNING, which the driver now sets before every call. It reads the marker the start step writes, which is what the status stood in for. - The failed-migration runner tagged its device by asking whether any task for it was still open — correct only because handle_task_result had already written this one DONE. It moves to on_finish, where that is true by construction rather than by accident. - The master-task roll-up becomes per-cycle upkeep. It reads every sub-task of a master anyway, so running it once a cycle covers status changes that finish no sub-task, and drops the redundant recomputation each sibling used to trigger. --- .../cluster_expansion/preconditions.py | 4 +- .../controllers/tasks_controller.py | 21 - .../services/migration_task_common.py | 228 +++++++++ simplyblock_core/services/task_runner_base.py | 33 +- .../services/tasks_runner_failed_migration.py | 193 +++----- .../services/tasks_runner_lvol_migration.py | 2 +- .../services/tasks_runner_migration.py | 442 +++++++----------- .../tasks_runner_new_dev_migration.py | 234 +++------- simplyblock_core/utils/__init__.py | 45 -- tests/unit/tasks/test_retry_ceiling.py | 6 +- tests/unit/tasks/test_runner_specs.py | 144 ++++++ tests/unit/tasks/test_task_runner_base.py | 34 ++ tests/unit/test_node_removal.py | 22 +- 13 files changed, 743 insertions(+), 665 deletions(-) create mode 100644 simplyblock_core/services/migration_task_common.py diff --git a/simplyblock_core/controllers/cluster_expansion/preconditions.py b/simplyblock_core/controllers/cluster_expansion/preconditions.py index 5188657ddf..479e92375e 100644 --- a/simplyblock_core/controllers/cluster_expansion/preconditions.py +++ b/simplyblock_core/controllers/cluster_expansion/preconditions.py @@ -25,7 +25,7 @@ runner defers its tasks while a cluster-expand task is open (they may be QUEUED — e.g. by an unexpected node outage mid-expansion — but never run before the expansion completes; see -``tasks_controller.defer_task_for_expansion``), ``shutdown_storage_node`` +``migration_task_common.require_active_cluster``), ``shutdown_storage_node`` refuses shutdowns during IN_EXPANSION, the executor holds a restart-phase gate on each donor (queueing create/delete/resize for the affected LVS), and the donors' outbound hublvol connections are dropped up-front (see @@ -78,7 +78,7 @@ #: Subset of the blocking families that DEFER on an open cluster-expand task #: (their runners suspend while the expansion is in progress — see -#: ``tasks_controller.defer_task_for_expansion``). A RESUME of an in-progress +#: ``migration_task_common.require_active_cluster``). A RESUME of an in-progress #: plan tolerates open tasks from these families: they are typically the #: recovery migrations queued by an unexpected node outage mid-expansion, #: and they wait for us, not the other way around (required order: expansion diff --git a/simplyblock_core/controllers/tasks_controller.py b/simplyblock_core/controllers/tasks_controller.py index ba81b08b07..d51cef08c5 100644 --- a/simplyblock_core/controllers/tasks_controller.py +++ b/simplyblock_core/controllers/tasks_controller.py @@ -870,27 +870,6 @@ def get_active_cluster_expand_task(cluster_id): return False -def defer_task_for_expansion(task): - """Suspend ``task`` if a cluster expansion is in progress. Returns True - when deferred. - - Migration-family runners call this right after their cluster-status - gate. The status gate alone is not enough: a node outage mid-expansion - suspends the cluster-expand task and restores the cluster status to - ACTIVE between its retries — without this gate the outage's recovery - migrations would start running in that window and then block the - expansion resume, inverting the required order (expansion completes - FIRST, then outage device migration, then expansion migration). - Deliberately does not consume a retry: this is a deferral, not a - failure.""" - if not get_active_cluster_expand_task(task.cluster_id): - return False - task.function_result = "cluster expansion in progress, deferring" - task.status = JobSchedule.STATUS_SUSPENDED - task.write_to_db(db.kv_store) - return True - - def get_active_node_tasks(cluster_id, node_id): tasks = db.get_job_tasks(cluster_id) out = [] diff --git a/simplyblock_core/services/migration_task_common.py b/simplyblock_core/services/migration_task_common.py new file mode 100644 index 0000000000..90d0f7393f --- /dev/null +++ b/simplyblock_core/services/migration_task_common.py @@ -0,0 +1,228 @@ +# coding=utf-8 +"""Shared pieces of the three device-migration runners. + +``tasks_runner_migration`` (FN_DEV_MIG), ``tasks_runner_new_dev_migration`` +(FN_NEW_DEV_MIG) and ``tasks_runner_failed_migration`` (FN_FAILED_DEV_MIG) all +drive the same data-plane operation — start a distr migration, then poll +``distr_migration_status`` until it settles — and differ only in what they +migrate and when they are allowed to start. +""" +from datetime import datetime, timezone + +from simplyblock_core import db_controller, utils +from simplyblock_core.controllers import tasks_controller +from simplyblock_core.models.nvme_device import NVMeDevice +from simplyblock_core.models.storage_node import StorageNode +from simplyblock_core.services.task_runner_base import ( + TaskAbort, + TaskDefer, + TaskProgress, + TaskRetry, + checkpoint, +) + +logger = utils.get_logger(__name__) + +db = db_controller.DBController() + +MIGRATION_WAIT_UNAVAILABLE_KEY = "wait_unavailable_before_retry" + +# A migration may only start once the cluster has been serving for a moment: +# a node that just came back is still settling its lvstore. +NODE_SETTLE_SEC = 60 + + +def migration_started(task): + """Whether this task has already issued its data-plane migration. + + The runners used to read this off ``task.status`` being RUNNING. That no + longer distinguishes anything — the driver moves a task to RUNNING before + calling the handler — so the marker the start step writes is used directly, + which is what the status was standing in for anyway. + """ + return "migration" in task.function_params + + +def require_active_cluster(task): + # IN_SHRINK is operable: draining data off the departing node is the node + # removal's own work, and removal blocks until every data device reaches + # FAILED_AND_MIGRATED — which only this family sets. Refusing here would + # deadlock the removal against the status it set itself. + cluster = db.get_cluster_by_id(task.cluster_id) + if not cluster.allows_operation(): + raise TaskRetry("cluster is not active, retrying") + + # Expansion-first ordering: no data migration runs while a cluster + # expansion is open — even between the expand task's retries, when the + # cluster status is momentarily ACTIVE. Without this, an outage's recovery + # migrations would start in that window and then block the expansion + # resume, inverting the required order (expansion completes FIRST, then + # outage device migration, then expansion migration). A deferral, not a + # failure: no retry consumed. + if tasks_controller.get_active_cluster_expand_task(task.cluster_id): + raise TaskDefer("cluster expansion in progress, deferring") + + +def require_node(task): + try: + return db.get_storage_node_by_id(task.node_id) + except KeyError: + raise TaskAbort(f"Node not found: {task.node_id}") + + +def nodes_settled(cluster_id, primaries_only=False): + """False while any node has been back for less than NODE_SETTLE_SEC. + + The three runners disagree on whether waiting for this costs a retry, so + the verdict is returned rather than raised. + """ + for node in db.get_storage_nodes_by_cluster_id(cluster_id): + if primaries_only and node.is_secondary_node: + continue + if not node.online_since: + continue + try: + settled_for = datetime.now(timezone.utc) - datetime.fromisoformat(node.online_since) + except Exception as e: + logger.error(f"Failed to get online since: {e}") + continue + if settled_for.total_seconds() < NODE_SETTLE_SEC: + return False + return True + + +def cluster_unavailable_state(cluster_id): + """Nodes and devices that are neither serving nor written off, as stable + ids — the set a migration waits on before (re)starting.""" + unavailable = [] + for node in db.get_storage_nodes_by_cluster_id(cluster_id): + if node.status in [StorageNode.STATUS_IN_CREATION, StorageNode.STATUS_REMOVED]: + continue + if node.status not in [StorageNode.STATUS_ONLINE, StorageNode.STATUS_SUSPENDED]: + unavailable.append(f"node:{node.get_id()}") + for dev in node.nvme_devices: + if dev.status in [NVMeDevice.STATUS_REMOVED, NVMeDevice.STATUS_FAILED_AND_MIGRATED]: + continue + if dev.status != NVMeDevice.STATUS_ONLINE: + unavailable.append(f"dev:{dev.get_id()}") + return sorted(unavailable) + + +def require_recovery_progress(task, unavailable): + """Hold a migration back while the cluster is degraded, releasing it only + when something actually recovers. + + Retrying against an unchanged set of unavailable nodes/devices just burns + the budget, so the set is recorded on the task and compared: an unchanged + (or grown) set defers, while any member coming back is the recovery event + that lets the migration proceed. + """ + previous = sorted(task.function_params.get(MIGRATION_WAIT_UNAVAILABLE_KEY, [])) + + if not unavailable: + if previous: + task.function_params.pop(MIGRATION_WAIT_UNAVAILABLE_KEY, None) + return + + recovered = set(previous) - set(unavailable) + task.function_params[MIGRATION_WAIT_UNAVAILABLE_KEY] = unavailable + if previous and recovered: + logger.info("Migration retry allowed after recovery event for task %s: %s", + task.uuid, sorted(recovered)) + return + + raise TaskDefer("waiting for unavailable nodes/devices to recover before " + f"restarting migration: {unavailable}") + + +def start_migration(task, start): + """Issue the data-plane migration and record that it was issued. + + ``start`` is the runner's own RPC call, returning falsy on failure. The + marker is checkpointed immediately: a crash between the RPC and the end of + the handler would otherwise lose it and start a second migration. + """ + try: + started = start() + except Exception as e: + logger.error(e) + started = False + if not started: + raise TaskRetry("Failed to start device migration task, retry later") + + return checkpoint(task, migration={"name": task.function_params["distr_name"]}) + + +def report_migration_status(task, res, allow_all_errors=False, allowed_error_codes=None): + """Translate a ``distr_migration_status`` poll into the task's outcome.""" + if not res: + raise TaskRetry("Failed to get mig status") + + allowed_error_codes = allowed_error_codes or [0] + res_data = res[0] + migration_status = res_data.get("status") + error_code = res_data.get("error", -1) + progress = res_data.get("progress", -1) + + if migration_status == "completed": + if error_code == 0: + task.function_result = "Done" + return + if error_code in allowed_error_codes or allow_all_errors: + task.function_result = f"mig completed with status: {error_code}" + return + # Drop the marker so the next attempt starts a fresh migration rather + # than polling the one that just errored. + del task.function_params['migration'] + raise TaskRetry(f"mig error: {error_code}, retrying") + + if migration_status == "failed": + raise TaskAbort(migration_status) + + if migration_status == "none": + del task.function_params['migration'] + raise TaskRetry("mig retry after restart") + + raise TaskProgress(f"Status: {migration_status}, progress:{progress}") + + +def no_sibling_migration(task): + """Eligibility: one migration at a time per node. + + Only gates a task that has not started yet — once its own migration is + running it is itself the sibling everything else waits on. + """ + if migration_started(task): + return True + return not tasks_controller.get_active_node_mig_task( + task.cluster_id, task.node_id, task.function_params.get("distr_name")) + + +def allow_all_migration_errors(cluster_id, statuses): + for node in db.get_storage_nodes_by_cluster_id(cluster_id): + for dev in node.nvme_devices: + if dev.status in statuses: + return True + return False + + +def poll_migration(task, rpc_client, allow_all_errors=False): + try: + res = rpc_client.distr_migration_status(**task.function_params["migration"]) + except (TaskAbort, TaskDefer, TaskProgress, TaskRetry): + raise + except Exception as e: + logger.error("Failed to get migration task status") + logger.exception(e) + raise TaskRetry("Failed to get migration status") + + report_migration_status(task, res, allow_all_errors=allow_all_errors) + + +def qos_high_priority(cluster_id): + return db.get_cluster_by_id(cluster_id).is_qos_set() + + +def sibling_eligibility(task, cluster): + """Spec-shaped wrapper of :func:`no_sibling_migration`.""" + return no_sibling_migration(task) diff --git a/simplyblock_core/services/task_runner_base.py b/simplyblock_core/services/task_runner_base.py index a638ed797c..99f53663cb 100644 --- a/simplyblock_core/services/task_runner_base.py +++ b/simplyblock_core/services/task_runner_base.py @@ -18,8 +18,11 @@ flow: - **return** (``None``) — the task is terminally complete → ``STATUS_DONE``. -- **raise** :class:`TaskDefer` — not terminal: still in progress, or blocked on - external state. Re-poll next cycle; **no retry consumed**; no backoff. +- **raise** :class:`TaskDefer` — cannot proceed yet, blocked on external state. + Suspend and re-poll next cycle; **no retry consumed**; no backoff. +- **raise** :class:`TaskProgress` — the work is under way and this poll found it + unfinished. As TaskDefer, but the task stays RUNNING; runners that gate + mutual exclusion on a RUNNING sibling depend on that. - **raise** :class:`TaskRetry` (or any other, unexpected ``Exception``) — a retryable failure. Suspend, **consume a retry**, and back off before the next attempt. @@ -65,8 +68,18 @@ class TaskDefer(Exception): - """Handler signal: the task is not terminal — still in progress, or blocked - on external state. Re-poll next cycle without consuming a retry.""" + """Handler signal: the task cannot proceed yet — blocked on external state. + Suspend and re-poll next cycle without consuming a retry.""" + + +class TaskProgress(Exception): + """Handler signal: the work is under way and this poll found it unfinished. + + Like :class:`TaskDefer` it consumes no retry, but the task stays RUNNING + rather than being suspended. That distinction is load-bearing: the + migration family gates mutual exclusion on a sibling task being RUNNING + (tasks_controller.get_active_node_mig_task), so suspending a migration + between polls would let a second one start on the same node.""" class TaskRetry(Exception): @@ -342,6 +355,8 @@ def _process(self, task: JobSchedule, cluster: Any) -> None: # second host claim and double-drive the task. with tasks_controller.task_lease_heartbeat(task): self.spec.handler(task) + except TaskProgress as e: + self._progress(task, str(e)) except TaskDefer as e: self._defer(task, str(e)) except TaskAbort as e: @@ -395,6 +410,16 @@ def _write_terminal(self, task: JobSchedule) -> None: logger.error(f"{self.spec.name}: task {task.uuid} on_finish failed: {e}") logger.exception(e) + def _progress(self, task: JobSchedule, reason: str) -> None: + if reason: + task.function_result = reason + # Status stays RUNNING — see TaskProgress. The commit still happens, to + # record the progress message and refresh the lease. + if self._cas(task, self._to(JobSchedule.STATUS_RUNNING)) is None: + self._forget(task.uuid) + return + self._clear_backoff(task.uuid) + def _defer(self, task: JobSchedule, reason: str) -> None: if reason: task.function_result = reason diff --git a/simplyblock_core/services/tasks_runner_failed_migration.py b/simplyblock_core/services/tasks_runner_failed_migration.py index df0f075e03..d95efdbab3 100644 --- a/simplyblock_core/services/tasks_runner_failed_migration.py +++ b/simplyblock_core/services/tasks_runner_failed_migration.py @@ -1,169 +1,86 @@ # coding=utf-8 import time -from datetime import datetime from simplyblock_core import db_controller, utils, constants from simplyblock_core.controllers import tasks_controller, device_controller -from simplyblock_core.models.cluster import Cluster from simplyblock_core.models.job_schedule import JobSchedule +from simplyblock_core.models.storage_node import StorageNode +from simplyblock_core.services import migration_task_common as mig +from simplyblock_core.services.task_runner_base import ( + RunnerSpec, + TaskAbort, + TaskRetry, + serve, +) +logger = utils.get_logger(__name__) - -from simplyblock_core.models.storage_node import StorageNode +# get DB controller +db = db_controller.DBController() def task_runner(task): - try: - snode = db.get_storage_node_by_id(task.node_id) - except KeyError: - task.status = JobSchedule.STATUS_DONE - task.function_result = f"Node not found: {task.node_id}" - task.write_to_db(db.kv_store) - return True - - cluster = db.get_cluster_by_id(task.cluster_id) - # Node removal DEPENDS on this runner: _decommission_node_devices blocks - # until every data device reaches FAILED_AND_MIGRATED, which only this - # runner sets — refusing IN_SHRINK deadlocks removal against its own status. - if cluster.status not in Cluster.OPERABLE_STATUSES: - task.function_result = "cluster is not active, retrying" - task.status = JobSchedule.STATUS_SUSPENDED - task.retry += 1 - task.write_to_db(db.kv_store) - return False - - # Expansion-first ordering: defer while a cluster expansion is open - # (see tasks_controller.defer_task_for_expansion). - if tasks_controller.defer_task_for_expansion(task): - return False - - if task.canceled: - task.function_result = "canceled" - task.status = JobSchedule.STATUS_DONE - task.write_to_db(db.kv_store) - return True - - if task.status in [JobSchedule.STATUS_NEW ,JobSchedule.STATUS_SUSPENDED]: - if task.status == JobSchedule.STATUS_NEW: - for node in db.get_storage_nodes_by_cluster_id(task.cluster_id): - if node.online_since: - try: - diff = datetime.now() - datetime.fromisoformat(node.online_since) - if diff.total_seconds() < 60: - task.function_result = "node is online < 1 min, retrying" - task.status = JobSchedule.STATUS_SUSPENDED - task.retry += 1 - task.write_to_db(db.kv_store) - return False - except Exception as e: - logger.error(f"Failed to get online since: {e}") - - task.status = JobSchedule.STATUS_RUNNING - task.write_to_db(db.kv_store) + snode = mig.require_node(task) + mig.require_active_cluster(task) + + if not mig.migration_started(task): + if not mig.nodes_settled(task.cluster_id): + raise TaskRetry("node is online < 1 min, retrying") if snode.status != StorageNode.STATUS_ONLINE: - task.function_result = "node is not online, retrying" - task.status = JobSchedule.STATUS_SUSPENDED - task.retry += 1 - task.write_to_db(db.kv_store) - return False - - active_f_task = tasks_controller.get_new_device_mig_task_for_device(task.cluster_id) - if active_f_task: - msg = "dev expansion task found, retry" - logger.info(msg) - task.function_result = msg - task.retry += 1 - task.status = JobSchedule.STATUS_SUSPENDED - task.write_to_db(db.kv_store) - return False + raise TaskRetry("node is not online, retrying") + + if tasks_controller.get_new_device_mig_task_for_device(task.cluster_id): + raise TaskRetry("dev expansion task found, retry") rpc_client = snode.rpc_client(timeout=5, retry=2) - if "migration" not in task.function_params: + + if not mig.migration_started(task): try: device = db.get_storage_device_by_id(task.device_id) except KeyError: - task.status = JobSchedule.STATUS_DONE - task.function_result = "Device not found" - task.write_to_db(db.kv_store) - return True - - distr_name = task.function_params["distr_name"] - - qos_high_priority = False - if db.get_cluster_by_id(snode.cluster_id).is_qos_set(): - qos_high_priority = True - try: - rsp = rpc_client.distr_migration_failure_start( - distr_name, device.cluster_device_order, qos_high_priority, job_size=constants.MIG_JOB_SIZE, jobs=constants.MIG_PARALLEL_JOBS) - except Exception as e: - logger.error(e) - rsp = False - if not rsp: - logger.error(f"Failed to start device migration task, storage_ID: {device.cluster_device_order}") - task.function_result = "Failed to start device migration task" - task.retry += 1 - task.status = JobSchedule.STATUS_SUSPENDED - task.write_to_db(db.kv_store) - return False - - task.function_params['migration'] = {"name": distr_name} - task.write_to_db(db.kv_store) + raise TaskAbort("Device not found") + + started = mig.start_migration(task, lambda: rpc_client.distr_migration_failure_start( + task.function_params["distr_name"], device.cluster_device_order, + mig.qos_high_priority(snode.cluster_id), + job_size=constants.MIG_JOB_SIZE, jobs=constants.MIG_PARALLEL_JOBS)) + if started is None: + raise TaskAbort("canceled while starting migration") + task = started time.sleep(3) - try: - if "migration" in task.function_params: - mig_info = task.function_params["migration"] - res = rpc_client.distr_migration_status(**mig_info) - out = utils.handle_task_result(task, res) - dev_failed_task = tasks_controller.get_failed_device_mig_task(task.cluster_id, task.device_id) - if not dev_failed_task: - device_controller.device_set_failed_and_migrated(task.device_id) + mig.poll_migration(task, rpc_client) - return out - except Exception as e: - logger.error("Failed to get migration task status") - logger.exception(e) - task.function_result = "Failed to get migration status" - task.retry += 1 - task.write_to_db(db.kv_store) - return False +def tag_device_migrated(task): + """The device's data now lives elsewhere, so record that on the device. + Runs from the driver's on_finish rather than inline: the check below asks + whether any failed-migration task for this device is still open, and this + task only stops counting as open once it has been written DONE. Inline it + would always find itself and never tag anything — which the old code got + away with only because its handle_task_result wrote DONE first. + """ + if not mig.migration_started(task): + return # never got as far as moving any data + if tasks_controller.get_failed_device_mig_task(task.cluster_id, task.device_id): + return + device_controller.device_set_failed_and_migrated(task.device_id) -logger = utils.get_logger(__name__) -# get DB controller -db = db_controller.DBController() +SPEC = RunnerSpec( + name="tasks-runner-failed-migration", + function_names=[JobSchedule.FN_FAILED_DEV_MIG], + handler=task_runner, + on_finish=tag_device_migrated, + is_eligible=mig.sibling_eligibility, + interval=3, +) def main(): - logger.info("Starting Tasks runner...") - while True: - time.sleep(3) - clusters = db.get_clusters() - if not clusters: - logger.error("No clusters found!") - else: - for cl in clusters: - tasks = db.get_job_tasks(cl.get_id(), reverse=False) - for task in tasks: - if task.function_name == JobSchedule.FN_FAILED_DEV_MIG: - if task.status in [JobSchedule.STATUS_NEW, JobSchedule.STATUS_SUSPENDED]: - active_task = tasks_controller.get_active_node_mig_task( - task.cluster_id, task.node_id, task.function_params["distr_name"]) - if active_task: - logger.info("task found on same node, retry") - continue - if task.status != JobSchedule.STATUS_DONE: - # get new task object because it could be changed from cancel task - task = db.get_task_by_id(task.uuid) - if not tasks_controller.claim_task(task): - logger.info(f"Failed-migration task {task.uuid} owned by another runner host; skipping") - continue - res = task_runner(task) - if not res: - time.sleep(3) + serve(SPEC) if __name__ == "__main__": diff --git a/simplyblock_core/services/tasks_runner_lvol_migration.py b/simplyblock_core/services/tasks_runner_lvol_migration.py index 769d9c0271..9941e6916e 100644 --- a/simplyblock_core/services/tasks_runner_lvol_migration.py +++ b/simplyblock_core/services/tasks_runner_lvol_migration.py @@ -2947,7 +2947,7 @@ def task_runner(task): # Expansion-first ordering: defer while a cluster expansion is open — # even between the expand task's retries, when the cluster status is - # momentarily ACTIVE (see tasks_controller.defer_task_for_expansion). + # momentarily ACTIVE (see migration_task_common.require_active_cluster). if tasks_controller.get_active_cluster_expand_task(task.cluster_id): return _suspend_task( task, migration, "cluster expansion in progress, deferring", diff --git a/simplyblock_core/services/tasks_runner_migration.py b/simplyblock_core/services/tasks_runner_migration.py index e2f645c27a..2054bf4e9a 100644 --- a/simplyblock_core/services/tasks_runner_migration.py +++ b/simplyblock_core/services/tasks_runner_migration.py @@ -1,154 +1,78 @@ # coding=utf-8 -import time -from datetime import datetime, timezone - from simplyblock_core import db_controller, utils, constants from simplyblock_core.controllers import tasks_events, tasks_controller, lvol_controller -from simplyblock_core.models.cluster import Cluster from simplyblock_core.models.job_schedule import JobSchedule from simplyblock_core.models.nvme_device import NVMeDevice from simplyblock_core.models.storage_node import StorageNode from simplyblock_core.release_upgrades import jc_compression_upgrade +from simplyblock_core.services import migration_task_common as mig +from simplyblock_core.services.task_runner_base import ( + RunnerSpec, + TaskAbort, + TaskDefer, + TaskRetry, + serve, +) logger = utils.get_logger(__name__) -MIGRATION_WAIT_UNAVAILABLE_KEY = "wait_unavailable_before_retry" +# get DB controller +db = db_controller.DBController() +MIGRATION_WAIT_UNAVAILABLE_KEY = mig.MIGRATION_WAIT_UNAVAILABLE_KEY -def _cluster_unavailable_state(cluster_id): - unavailable = [] + +def _online_device_count(cluster_id, primaries_only=False): + online = 0 for node in db.get_storage_nodes_by_cluster_id(cluster_id): - if node.status in [StorageNode.STATUS_IN_CREATION, StorageNode.STATUS_REMOVED]: + if primaries_only and node.is_secondary_node: continue - if node.status not in [StorageNode.STATUS_ONLINE, StorageNode.STATUS_SUSPENDED]: - unavailable.append(f"node:{node.get_id()}") for dev in node.nvme_devices: - if dev.status in [NVMeDevice.STATUS_REMOVED, NVMeDevice.STATUS_FAILED_AND_MIGRATED]: - continue - if dev.status != NVMeDevice.STATUS_ONLINE: - unavailable.append(f"dev:{dev.get_id()}") - return sorted(unavailable) + if dev.status == NVMeDevice.STATUS_ONLINE: + online += 1 + return online -def _migration_retry_allowed(task, unavailable): - previous = sorted(task.function_params.get(MIGRATION_WAIT_UNAVAILABLE_KEY, [])) - if not unavailable: - if previous: - task.function_params.pop(MIGRATION_WAIT_UNAVAILABLE_KEY, None) - task.write_to_db(db.kv_store) - return True - - recovered = set(previous) - set(unavailable) - if previous and recovered: - task.function_params[MIGRATION_WAIT_UNAVAILABLE_KEY] = unavailable - task.write_to_db(db.kv_store) - logger.info( - "Migration retry allowed after recovery event for task %s: %s", - task.uuid, - sorted(recovered), - ) - return True +def _wait_for_cluster_recovery(task): + """Gate a not-yet-started migration on the cluster being whole enough. - task.function_params[MIGRATION_WAIT_UNAVAILABLE_KEY] = unavailable - task.function_result = ( - "waiting for unavailable nodes/devices to recover before restarting migration: " - f"{unavailable}" - ) - task.status = JobSchedule.STATUS_SUSPENDED - task.write_to_db(db.kv_store) - return False + Whether waiting costs a retry depends on why: with nothing unavailable the + hold-up is transient and counts against the budget, but while nodes or + devices are down it is the recovery we are waiting on, and burning retries + against it would terminate the migration before the cluster came back. + """ + unavailable = mig.cluster_unavailable_state(task.cluster_id) + if not unavailable: + raise TaskRetry(task.function_result or "waiting to start migration, retrying") + mig.require_recovery_progress(task, unavailable) def task_runner(task): - - task = db.get_task_by_id(task.uuid) - try: - snode = db.get_storage_node_by_id(task.node_id) - except KeyError: - task.status = JobSchedule.STATUS_DONE - task.function_result = f"Node not found: {task.node_id}" - task.write_to_db(db.kv_store) - return True - - if task.canceled: - task.function_result = "canceled" - task.status = JobSchedule.STATUS_DONE - task.write_to_db(db.kv_store) - return True + snode = mig.require_node(task) if snode.status not in [StorageNode.STATUS_ONLINE, StorageNode.STATUS_SUSPENDED]: task.function_result = "node is not online, retrying" - task.status = JobSchedule.STATUS_SUSPENDED - unavailable = _cluster_unavailable_state(task.cluster_id) - if not unavailable: - task.retry += 1 - task.write_to_db(db.kv_store) - else: - _migration_retry_allowed(task, unavailable) - return False - - cluster = db.get_cluster_by_id(task.cluster_id) - if cluster.status not in Cluster.OPERABLE_STATUSES: - task.function_result = "cluster is not active, retrying" - task.status = JobSchedule.STATUS_SUSPENDED - task.retry += 1 - task.write_to_db(db.kv_store) - return False - - # Expansion-first ordering: no data migration runs while a cluster - # expansion is open — even between the expand task's retries, when the - # cluster status is momentarily ACTIVE (see defer_task_for_expansion). - if tasks_controller.defer_task_for_expansion(task): - return False + _wait_for_cluster_recovery(task) + raise TaskDefer("node is not online, retrying") + + mig.require_active_cluster(task) if tasks_controller.get_active_lvol_migration(task.node_id): - task.function_result = "LVol migration tasks found, retrying" - task.status = JobSchedule.STATUS_SUSPENDED - task.retry += 1 - task.write_to_db(db.kv_store) - return False - - if task.status in [JobSchedule.STATUS_NEW, JobSchedule.STATUS_SUSPENDED]: - current_online_devices = 0 - unavailable = _cluster_unavailable_state(task.cluster_id) - for node in db.get_storage_nodes_by_cluster_id(task.cluster_id): - if node.is_secondary_node: # pass - continue - for dev in node.nvme_devices: - if dev.status == NVMeDevice.STATUS_ONLINE: - current_online_devices += 1 - if node.status == StorageNode.STATUS_ONLINE and node.online_since: - try: - diff = datetime.now(timezone.utc) - datetime.fromisoformat(node.online_since) - if diff.total_seconds() < 60: - task.function_result = "node is online < 1 min, retrying" - task.status = JobSchedule.STATUS_SUSPENDED - task.write_to_db(db.kv_store) - return False - except Exception as e: - logger.error(f"Failed to get online since: {e}") - - migration_devices = 0 - if "migration_devices" in task.function_params: - migration_devices = task.function_params["migration_devices"] - - if current_online_devices < migration_devices: - task.function_result = f"only {current_online_devices} devices online, waiting for more devices to be online" - task.status = JobSchedule.STATUS_SUSPENDED - if not unavailable: - task.retry += 1 - task.write_to_db(db.kv_store) - else: - _migration_retry_allowed(task, unavailable) - return False + raise TaskRetry("LVol migration tasks found, retrying") - if not _migration_retry_allowed(task, unavailable): - return False + if not mig.migration_started(task): + if not mig.nodes_settled(task.cluster_id, primaries_only=True): + raise TaskDefer("node is online < 1 min, retrying") - task.status = JobSchedule.STATUS_RUNNING - task.function_result = "" - task.write_to_db(db.kv_store) + online_devices = _online_device_count(task.cluster_id, primaries_only=True) + wanted = task.function_params.get("migration_devices", 0) + if online_devices < wanted: + task.function_result = (f"only {online_devices} devices online, waiting for " + f"more devices to be online") + _wait_for_cluster_recovery(task) + raise TaskDefer(task.function_result) + mig.require_recovery_progress(task, mig.cluster_unavailable_state(task.cluster_id)) rpc_client = snode.rpc_client(timeout=5, retry=2) @@ -156,178 +80,138 @@ def task_runner(task): # Migration IO triggers auto-leader promotion in the data plane, so # starting migration on a non-leader causes a split-brain write conflict. if not snode.is_secondary_node and not lvol_controller.is_node_leader(snode, snode.lvstore): - msg = f"Node {snode.get_id()} is not the leader for {snode.lvstore}, deferring migration" - logger.warning(msg) - task.function_result = msg - task.status = JobSchedule.STATUS_SUSPENDED - task.retry += 1 - task.write_to_db(db.kv_store) - return False - - if "migration" not in task.function_params: - current_online_devices = 0 - for node in db.get_storage_nodes_by_cluster_id(task.cluster_id): - for dev in node.nvme_devices: - if dev.status == NVMeDevice.STATUS_ONLINE: - current_online_devices += 1 - - distr_name = task.function_params["distr_name"] - - qos_high_priority = False - if db.get_cluster_by_id(snode.cluster_id).is_qos_set(): - qos_high_priority = True - try: - rsp = rpc_client.distr_migration_expansion_start(distr_name, qos_high_priority, job_size=constants.MIG_JOB_SIZE, - jobs=constants.MIG_PARALLEL_JOBS) - except Exception as e: - logger.error(e) - rsp = False - if not rsp: - msg = "Failed to start device migration task, retry later" - logger.error(msg) - task.function_result =msg - task.status = JobSchedule.STATUS_SUSPENDED - unavailable = _cluster_unavailable_state(task.cluster_id) - if not unavailable: - task.retry += 1 - task.write_to_db(db.kv_store) - else: - _migration_retry_allowed(task, unavailable) - return True - task.function_params['migration'] = {"name": distr_name} - task.function_params['migration_devices'] = current_online_devices - task.write_to_db(db.kv_store) - - try: - if "migration" in task.function_params: - allow_all_errors = False - for node in db.get_storage_nodes_by_cluster_id(task.cluster_id): - for dev in node.nvme_devices: - if dev.status in [NVMeDevice.STATUS_READONLY, NVMeDevice.STATUS_CANNOT_ALLOCATE, NVMeDevice.STATUS_FAILED]: - allow_all_errors = True - break - - mig_info = task.function_params["migration"] - res = rpc_client.distr_migration_status(**mig_info) - return utils.handle_task_result(task, res, allow_all_errors=allow_all_errors) - except Exception as e: - logger.error("Failed to get migration task status") - logger.exception(e) - task.function_result = "Failed to get migration status" - - task.retry += 1 - task.write_to_db(db.kv_store) - return False + raise TaskRetry(f"Node {snode.get_id()} is not the leader for {snode.lvstore}, " + f"deferring migration") + + if not mig.migration_started(task): + # Recorded alongside the start so a later poll can tell how much of the + # cluster the migration was sized against. + task.function_params["migration_devices"] = _online_device_count(task.cluster_id) + started = mig.start_migration(task, lambda: rpc_client.distr_migration_expansion_start( + task.function_params["distr_name"], mig.qos_high_priority(snode.cluster_id), + job_size=constants.MIG_JOB_SIZE, jobs=constants.MIG_PARALLEL_JOBS)) + if started is None: + raise TaskAbort("canceled while starting migration") + task = started + + mig.poll_migration(task, rpc_client, allow_all_errors=mig.allow_all_migration_errors( + task.cluster_id, [NVMeDevice.STATUS_READONLY, + NVMeDevice.STATUS_CANNOT_ALLOCATE, + NVMeDevice.STATUS_FAILED])) + + +def _is_eligible(task, cluster): + """One migration at a time per node and distr. + + Beyond the shared sibling rule, a device migration also waits behind a + SUSPENDED new-device migration for the same distr: that one is mid-flight + on the data plane even while its task is parked, and starting a second + migration over the same distr would collide with it. + """ + if mig.migration_started(task): + return True + distr_name = task.function_params.get("distr_name") + for sibling in db.get_job_tasks(task.cluster_id): + if sibling.function_name not in [JobSchedule.FN_FAILED_DEV_MIG, + JobSchedule.FN_DEV_MIG, + JobSchedule.FN_NEW_DEV_MIG]: + continue + if sibling.node_id != task.node_id or sibling.canceled: + continue + if sibling.function_params.get("distr_name") != distr_name: + continue + if sibling.status == JobSchedule.STATUS_RUNNING: + return False + if (sibling.status == JobSchedule.STATUS_SUSPENDED + and sibling.function_name == JobSchedule.FN_NEW_DEV_MIG): + return False + return True -# get DB controller -db = db_controller.DBController() +def update_master_tasks(cluster_id): + """Roll every master task's sub-task statuses up into it. -def update_master_task(task, cl): - master_task = None - tasks = {t.uuid: t for t in db.get_job_tasks(cl.get_id(), reverse=False)} - for t in tasks.values(): - if task.uuid in t.sub_tasks: - master_task = t - break + Per cycle rather than per sub-task attempt: the roll-up reads all of a + master's sub-tasks anyway, so running it once a cycle both covers every + status change (not only the attempts that finish a sub-task) and drops the + redundant re-computation each sibling used to trigger. + """ + tasks = {t.uuid: t for t in db.get_job_tasks(cluster_id, reverse=False)} + for master_task in list(tasks.values()): + if master_task.sub_tasks: + _roll_up(master_task, tasks) - def _set_master_task_status(master_task, status): - if master_task.status != status: - logger.info(f"_set_master_task_status: {status}") - master_task.status = status - master_task.function_result = status - master_task.write_to_db(db.kv_store) - tasks_events.task_updated(master_task) +def _roll_up(master_task, tasks): status_map = { JobSchedule.STATUS_DONE: 0, JobSchedule.STATUS_NEW: 0, JobSchedule.STATUS_SUSPENDED: 0, JobSchedule.STATUS_RUNNING: 0, } - if master_task: - for sub_task_id in master_task.sub_tasks: - sub_task = tasks[sub_task_id] - status_map[sub_task.status] = status_map.get(sub_task.status, 0) + 1 - - logger.info(f"master_task.sub_tasks: {len(master_task.sub_tasks)}") - logger.info(f"status_map: {status_map}") - - if status_map[JobSchedule.STATUS_DONE] == len(master_task.sub_tasks): # all tasks done - _set_master_task_status(master_task, JobSchedule.STATUS_DONE) - elif status_map[JobSchedule.STATUS_NEW] == len(master_task.sub_tasks): # all tasks new - _set_master_task_status(master_task, JobSchedule.STATUS_NEW) - elif status_map[JobSchedule.STATUS_SUSPENDED] == len(master_task.sub_tasks): # all tasks suspended - _set_master_task_status(master_task, JobSchedule.STATUS_SUSPENDED) - else: # set running - _set_master_task_status(master_task, JobSchedule.STATUS_RUNNING) - return True + for sub_task_id in master_task.sub_tasks: + sub_task = tasks.get(sub_task_id) + if sub_task is None: + return + status_map[sub_task.status] = status_map.get(sub_task.status, 0) + 1 + + total = len(master_task.sub_tasks) + for status in (JobSchedule.STATUS_DONE, JobSchedule.STATUS_NEW, JobSchedule.STATUS_SUSPENDED): + if status_map[status] == total: + rolled_up = status + break + else: + rolled_up = JobSchedule.STATUS_RUNNING + + if master_task.status == rolled_up: + return + logger.info(f"_set_master_task_status: {rolled_up}") + master_task.status = rolled_up + master_task.function_result = rolled_up + master_task.write_to_db(db.kv_store) + tasks_events.task_updated(master_task) + + +def resume_jc_compression(task): + """A finished migration frees the node for JC compression again.""" + if tasks_controller.get_active_node_tasks(task.cluster_id, task.node_id): + return + + # Release-upgrade guard (remove with the jc_compression_upgrade plugin): + # resumes are held until `cluster upgrade-complete`. + if jc_compression_upgrade.resume_is_held(db.get_cluster_by_id(task.cluster_id)): + logger.info("JC compression resume held: cluster upgrade in progress") + return + + logger.info("no task found on same node, resuming compression") + node = db.get_storage_node_by_id(task.node_id) + for peer in db.get_storage_nodes_by_cluster_id(node.cluster_id): + if peer.status not in [StorageNode.STATUS_ONLINE, StorageNode.STATUS_SUSPENDED]: + logger.warning("Not all nodes are online, can not resume JC compression") + try: + _, err = node.rpc_client(timeout=5, retry=2).jc_suspend_compression( + jm_vuid=node.jm_vuid, suspend=False) + if err: + logger.info("Failed to resume JC compression adding task...") + tasks_controller.add_jc_comp_resume_task(task.cluster_id, task.node_id, node.jm_vuid) + except Exception as e: + logger.error(e) + + +SPEC = RunnerSpec( + name="tasks-runner-migration", + function_names=[JobSchedule.FN_DEV_MIG], + handler=task_runner, + on_finish=resume_jc_compression, + on_cycle=lambda cluster: update_master_tasks(cluster.get_id()), + is_eligible=_is_eligible, + interval=3, +) def main(): - logger.info("Starting Tasks runner...") - while True: - clusters = db.get_clusters() - if not clusters: - logger.error("No clusters found!") - else: - for cl in clusters: - tasks = db.get_job_tasks(cl.get_id(), reverse=False) - for task in tasks: - if task.function_name == JobSchedule.FN_DEV_MIG and task.status != JobSchedule.STATUS_DONE: - task = db.get_task_by_id(task.uuid) - if task.status in [JobSchedule.STATUS_NEW, JobSchedule.STATUS_SUSPENDED]: - active_task = False - suspended_task= False - for t in db.get_job_tasks(task.cluster_id): - if t.function_name in [JobSchedule.FN_FAILED_DEV_MIG, JobSchedule.FN_DEV_MIG, - JobSchedule.FN_NEW_DEV_MIG] and t.node_id == task.node_id: - if "distr_name" in t.function_params and t.function_params[ - "distr_name"] == task.function_params['distr_name'] and t.canceled is False: - if t.status == JobSchedule.STATUS_RUNNING: - active_task = True - elif t.status == JobSchedule.STATUS_SUSPENDED and t.function_name == JobSchedule.FN_NEW_DEV_MIG: - suspended_task = True - if active_task and suspended_task: - break - if active_task or suspended_task: - logger.info("task found on same node, retry") - continue - elif task.status == JobSchedule.STATUS_RUNNING: - pass - - # Lease gate: skip a task another live runner host owns. - if not tasks_controller.claim_task(task): - logger.info(f"Migration task {task.uuid} owned by another runner host; skipping") - continue - with tasks_controller.task_lease_heartbeat(task): - res = task_runner(task) - update_master_task(task, cl) - if res: - node_task = tasks_controller.get_active_node_tasks(task.cluster_id, task.node_id) - # Release-upgrade guard (remove with the - # jc_compression_upgrade plugin): resumes are - # held until `cluster upgrade-complete`. - if not node_task and jc_compression_upgrade.resume_is_held(cl): - logger.info("JC compression resume held: cluster upgrade in progress") - elif not node_task: - logger.info("no task found on same node, resuming compression") - node = db.get_storage_node_by_id(task.node_id) - for n in db.get_storage_nodes_by_cluster_id(node.cluster_id): - if n.status not in [StorageNode.STATUS_ONLINE, StorageNode.STATUS_SUSPENDED]: - logger.warning("Not all nodes are online, can not resume JC compression") - continue - rpc_client = node.rpc_client(timeout=5, retry=2) - try: - ret, err = rpc_client.jc_suspend_compression(jm_vuid=node.jm_vuid, suspend=False) - if err: - logger.info("Failed to resume JC compression adding task...") - tasks_controller.add_jc_comp_resume_task(task.cluster_id, task.node_id, node.jm_vuid) - except Exception as e: - logger.error(e) - - time.sleep(3) + serve(SPEC) if __name__ == "__main__": diff --git a/simplyblock_core/services/tasks_runner_new_dev_migration.py b/simplyblock_core/services/tasks_runner_new_dev_migration.py index a74af6b0ce..1a49b9632d 100644 --- a/simplyblock_core/services/tasks_runner_new_dev_migration.py +++ b/simplyblock_core/services/tasks_runner_new_dev_migration.py @@ -1,197 +1,101 @@ # coding=utf-8 import time -from datetime import datetime, timezone from simplyblock_core import db_controller, utils, constants -from simplyblock_core.controllers import tasks_controller -from simplyblock_core.models.cluster import Cluster from simplyblock_core.models.job_schedule import JobSchedule +from simplyblock_core.models.nvme_device import NVMeDevice +from simplyblock_core.models.storage_node import StorageNode +from simplyblock_core.services import migration_task_common as mig +from simplyblock_core.services.task_runner_base import ( + RunnerSpec, + TaskAbort, + TaskDefer, + TaskRetry, + serve, +) +logger = utils.get_logger(__name__) +# get DB controller +db = db_controller.DBController() -from simplyblock_core.models.nvme_device import NVMeDevice -from simplyblock_core.models.storage_node import StorageNode + +def _recovery_migration_open(cluster_id): + """Recovery-before-expansion priority: the expansion migration (this task + family, queued when the role rebalance completes) must not run while any + outage-recovery data migration is open — an unexpected node outage during + the expansion queues those, and the required order is: expansion completes + -> outage device migration drains -> expansion migration runs.""" + for task in db.get_job_tasks(cluster_id): + if (task.function_name in (JobSchedule.FN_DEV_MIG, JobSchedule.FN_FAILED_DEV_MIG) + and task.status != JobSchedule.STATUS_DONE + and task.canceled is False): + return task + return None def task_runner(task): - try: - snode = db.get_storage_node_by_id(task.node_id) - except KeyError: - task.status = JobSchedule.STATUS_DONE - task.function_result = f"Node not found: {task.node_id}" - task.write_to_db(db.kv_store) - return True - - if task.canceled: - task.function_result = "canceled" - task.status = JobSchedule.STATUS_DONE - task.write_to_db(db.kv_store) - return True + snode = mig.require_node(task) if snode.status != StorageNode.STATUS_ONLINE: - task.function_result = "node is not online, retrying" - task.retry += 1 - task.status = JobSchedule.STATUS_SUSPENDED - task.write_to_db(db.kv_store) - return False - - cluster = db.get_cluster_by_id(task.cluster_id) - if cluster.status not in Cluster.OPERABLE_STATUSES: - task.function_result = "cluster is not active, retrying" - task.status = JobSchedule.STATUS_SUSPENDED - task.retry += 1 - task.write_to_db(db.kv_store) - return False - - # Expansion-first ordering: defer while a cluster expansion is open - # (see tasks_controller.defer_task_for_expansion). - if tasks_controller.defer_task_for_expansion(task): - return False - - # Recovery-before-expansion priority: the expansion migration (this - # task family, queued when the role rebalance completes) must not run - # while any outage-recovery data migration is open — an unexpected - # node outage during the expansion queues those, and the required - # order is: expansion completes -> outage device migration drains -> - # expansion migration runs. Deferral, not failure: no retry consumed. - for t in db.get_job_tasks(task.cluster_id): - if t.function_name in (JobSchedule.FN_DEV_MIG, JobSchedule.FN_FAILED_DEV_MIG) \ - and t.status != JobSchedule.STATUS_DONE and t.canceled is False: - task.function_result = ( - f"deferring: recovery migration {t.uuid} ({t.function_name}) is open") - task.status = JobSchedule.STATUS_SUSPENDED - task.write_to_db(db.kv_store) - return False - - if task.status in [JobSchedule.STATUS_NEW, JobSchedule.STATUS_SUSPENDED]: - for node in db.get_storage_nodes_by_cluster_id(task.cluster_id): - if node.is_secondary_node: # pass - continue - - if node.online_since: - try: - diff = datetime.now(timezone.utc) - datetime.fromisoformat(node.online_since) - if diff.total_seconds() < 60: - task.function_result = "node is online < 1 min, retrying" - task.status = JobSchedule.STATUS_SUSPENDED - task.retry += 1 - task.write_to_db(db.kv_store) - return False - except Exception as e: - logger.error(f"Failed to get online since: {e}") - - task.function_result = JobSchedule.STATUS_RUNNING - task.status = JobSchedule.STATUS_RUNNING - task.write_to_db(db.kv_store) + raise TaskRetry("node is not online, retrying") + + mig.require_active_cluster(task) + open_recovery = _recovery_migration_open(task.cluster_id) + if open_recovery is not None: + # Deferral, not failure: no retry consumed. + raise TaskDefer(f"deferring: recovery migration {open_recovery.uuid} " + f"({open_recovery.function_name}) is open") + if not mig.migration_started(task): + if not mig.nodes_settled(task.cluster_id, primaries_only=True): + raise TaskRetry("node is online < 1 min, retrying") rpc_client = snode.rpc_client(timeout=5, retry=2) - all_devs_online_or_failed = True - for node in db.get_storage_nodes_by_cluster_id(task.cluster_id): - for dev in node.nvme_devices: - if dev.status not in [NVMeDevice.STATUS_ONLINE, - NVMeDevice.STATUS_FAILED_AND_MIGRATED, - NVMeDevice.STATUS_CANNOT_ALLOCATE]: - all_devs_online_or_failed = False - break - if "migration" not in task.function_params: - if not all_devs_online_or_failed: - task.function_result = "Some devs are offline, retrying" - task.retry += 1 - task.status = JobSchedule.STATUS_SUSPENDED - task.write_to_db(db.kv_store) - return False + if not mig.migration_started(task): + if not _all_devices_online_or_written_off(task.cluster_id): + raise TaskRetry("Some devs are offline, retrying") try: - device = db.get_storage_device_by_id(task.device_id) + db.get_storage_device_by_id(task.device_id) except KeyError: - task.status = JobSchedule.STATUS_DONE - task.function_result = "Device not found" - task.write_to_db(db.kv_store) - return True - - distr_name = task.function_params["distr_name"] - - qos_high_priority = False - if db.get_cluster_by_id(snode.cluster_id).is_qos_set(): - qos_high_priority = True - try: - rsp = rpc_client.distr_migration_expansion_start( - distr_name, qos_high_priority, job_size=constants.MIG_JOB_SIZE,jobs=constants.MIG_PARALLEL_JOBS) - except Exception as e: - logger.error(f"Failed to start migration : {e}") - rsp = False - if not rsp: - logger.error(f"Failed to start device migration task, storage_ID: {device.cluster_device_order}") - task.function_result = "Failed to start device migration task" - task.status = JobSchedule.STATUS_SUSPENDED - task.retry += 1 - task.write_to_db(db.kv_store) - return False - - task.function_params['migration'] = { - "name": distr_name - } - task.write_to_db(db.kv_store) + raise TaskAbort("Device not found") + + started = mig.start_migration(task, lambda: rpc_client.distr_migration_expansion_start( + task.function_params["distr_name"], mig.qos_high_priority(snode.cluster_id), + job_size=constants.MIG_JOB_SIZE, jobs=constants.MIG_PARALLEL_JOBS)) + if started is None: + raise TaskAbort("canceled while starting migration") + task = started time.sleep(3) - try: - if "migration" in task.function_params: - allow_all_errors = False - for node in db.get_storage_nodes_by_cluster_id(task.cluster_id): - for dev in node.nvme_devices: - if dev.status in [NVMeDevice.STATUS_READONLY, NVMeDevice.STATUS_CANNOT_ALLOCATE]: - allow_all_errors = True - break - - mig_info = task.function_params["migration"] - res = rpc_client.distr_migration_status(**mig_info) - return utils.handle_task_result(task, res, allow_all_errors=allow_all_errors) - except Exception as e: - logger.error("Failed to get migration task status") - logger.exception(e) - task.function_result = "Failed to get migration status" - - task.retry += 1 - task.write_to_db(db.kv_store) - return False + mig.poll_migration(task, rpc_client, allow_all_errors=mig.allow_all_migration_errors( + task.cluster_id, [NVMeDevice.STATUS_READONLY, NVMeDevice.STATUS_CANNOT_ALLOCATE])) -logger = utils.get_logger(__name__) + +def _all_devices_online_or_written_off(cluster_id): + for node in db.get_storage_nodes_by_cluster_id(cluster_id): + for dev in node.nvme_devices: + if dev.status not in [NVMeDevice.STATUS_ONLINE, + NVMeDevice.STATUS_FAILED_AND_MIGRATED, + NVMeDevice.STATUS_CANNOT_ALLOCATE]: + return False + return True -# get DB controller -db = db_controller.DBController() +SPEC = RunnerSpec( + name="tasks-runner-new-dev-migration", + function_names=[JobSchedule.FN_NEW_DEV_MIG], + handler=task_runner, + is_eligible=mig.sibling_eligibility, + interval=3, +) def main(): - logger.info("Starting Tasks runner...") - while True: - time.sleep(3) - clusters = db.get_clusters() - if not clusters: - logger.error("No clusters found!") - else: - for cl in clusters: - tasks = db.get_job_tasks(cl.get_id(), reverse=False) - for task in tasks: - if task.function_name == JobSchedule.FN_NEW_DEV_MIG: - if task.status in [JobSchedule.STATUS_NEW, JobSchedule.STATUS_SUSPENDED]: - active_task = tasks_controller.get_active_node_mig_task( - task.cluster_id, task.node_id, task.function_params["distr_name"]) - if active_task: - logger.info("task found on same node, retry") - continue - if task.status != JobSchedule.STATUS_DONE: - # get new task object because it could be changed from cancel task - task = db.get_task_by_id(task.uuid) - if not tasks_controller.claim_task(task): - logger.info(f"New-device migration task {task.uuid} owned by another runner host; skipping") - continue - res = task_runner(task) - if not res: - time.sleep(2) + serve(SPEC) if __name__ == "__main__": diff --git a/simplyblock_core/utils/__init__.py b/simplyblock_core/utils/__init__.py index 474e9dedb6..3eb60d0b2b 100644 --- a/simplyblock_core/utils/__init__.py +++ b/simplyblock_core/utils/__init__.py @@ -32,7 +32,6 @@ from simplyblock_core import constants from simplyblock_core import shell_utils -from simplyblock_core.models.job_schedule import JobSchedule from simplyblock_core.models.nvme_device import NVMeDevice from simplyblock_web import node_utils @@ -872,50 +871,6 @@ def strfdelta_seconds(remainder: int) -> str: return out.strip() -def handle_task_result(task: JobSchedule, res: dict, allowed_error_codes=None, allow_all_errors=False): - if res: - if not allowed_error_codes: - allowed_error_codes = [0] - - res_data = res[0] - migration_status = res_data.get("status") - error_code = res_data.get("error", -1) - progress = res_data.get("progress", -1) - if migration_status == "completed": - if error_code == 0: - task.function_result = "Done" - task.status = JobSchedule.STATUS_DONE - elif error_code in allowed_error_codes or allow_all_errors: - task.function_result = f"mig completed with status: {error_code}" - task.status = JobSchedule.STATUS_DONE - else: - task.function_result = f"mig error: {error_code}, retrying" - task.retry += 1 - task.status = JobSchedule.STATUS_SUSPENDED - del task.function_params['migration'] - - task.write_to_db() - return True - - elif migration_status == "failed": - task.status = JobSchedule.STATUS_DONE - task.function_result = migration_status - task.write_to_db() - return True - - elif migration_status == "none": - task.function_result = "mig retry after restart" - task.retry += 1 - task.status = JobSchedule.STATUS_SUSPENDED - del task.function_params['migration'] - task.write_to_db() - return True - - else: - task.function_result = f"Status: {migration_status}, progress:{progress}" - task.write_to_db() - else: - logger.error("Failed to get mig status") logger = get_logger(__name__) diff --git a/tests/unit/tasks/test_retry_ceiling.py b/tests/unit/tasks/test_retry_ceiling.py index 70a4b00bc2..6e80bfba01 100644 --- a/tests/unit/tasks/test_retry_ceiling.py +++ b/tests/unit/tasks/test_retry_ceiling.py @@ -269,6 +269,9 @@ def _get_node(node_id): "tasks_runner_cluster_expand.py", "tasks_runner_node_add.py", "tasks_runner_restart.py", + "tasks_runner_migration.py", + "tasks_runner_new_dev_migration.py", + "tasks_runner_failed_migration.py", } # Runners that increment task.retry but are intentionally UNBOUNDED: the @@ -276,9 +279,6 @@ def _get_node(node_id): # recovery (see _migration_retry_allowed) rather than a fixed count. Value is # the reason, surfaced in the skip message. INTENTIONALLY_UNBOUNDED = { - "tasks_runner_migration.py": "created with max_retry=-1; retry gated on resource recovery", - "tasks_runner_failed_migration.py": "created with max_retry=-1; retry gated on resource recovery", - "tasks_runner_new_dev_migration.py": "created with max_retry=-1; retry gated on resource recovery", "tasks_runner_lvol_migration.py": "created with max_retry=-1; retry gated on resource recovery", "tasks_runner_node_removal.py": "created with max_retry=-1; multi-hour removal gated on failure-migration completion", } diff --git a/tests/unit/tasks/test_runner_specs.py b/tests/unit/tasks/test_runner_specs.py index 34bcf85c6c..53074ebba0 100644 --- a/tests/unit/tasks/test_runner_specs.py +++ b/tests/unit/tasks/test_runner_specs.py @@ -346,3 +346,147 @@ def test_cluster_expand_retries_an_incomplete_phase(cluster_expand): def test_cluster_expand_aborts_without_a_node(cluster_expand): with pytest.raises(trb.TaskAbort): cluster_expand.SPEC.handler(_task()) + + +# -- migration family ------------------------------------------------------- + +@pytest.fixture +def mig(monkeypatch): + import simplyblock_core.services.migration_task_common as common + monkeypatch.setattr(common, "db", MagicMock()) + monkeypatch.setattr(common, "tasks_controller", MagicMock()) + common.tasks_controller.get_active_node_mig_task.return_value = False + return common + + +def _status(state, error=0, progress=50): + return [{"status": state, "error": error, "progress": progress}] + + +def test_migration_completion_finishes_the_task(mig): + task = _task(distr_name="distr-1") + assert mig.report_migration_status(task, _status("completed")) is None + assert task.function_result == "Done" + + +def test_migration_in_progress_keeps_the_task_running(mig): + """Not a defer: suspending between polls would drop the RUNNING status + that the family's mutual exclusion is keyed on.""" + with pytest.raises(trb.TaskProgress): + mig.report_migration_status(_task(), _status("in_progress")) + + +def test_migration_failure_is_terminal(mig): + with pytest.raises(trb.TaskAbort): + mig.report_migration_status(_task(), _status("failed")) + + +def test_migration_error_restarts_from_scratch(mig): + """A disallowed error code drops the marker so the next attempt issues a + fresh migration instead of polling the one that errored.""" + task = _task(migration={"name": "distr-1"}) + with pytest.raises(trb.TaskRetry): + mig.report_migration_status(task, _status("completed", error=7)) + assert "migration" not in task.function_params + + +def test_migration_error_tolerated_when_devices_are_degraded(mig): + task = _task(migration={"name": "distr-1"}) + assert mig.report_migration_status(task, _status("completed", error=7), + allow_all_errors=True) is None + assert task.function_params["migration"] == {"name": "distr-1"} + + +def test_missing_migration_restarts_from_scratch(mig): + task = _task(migration={"name": "distr-1"}) + with pytest.raises(trb.TaskRetry): + mig.report_migration_status(task, _status("none")) + assert "migration" not in task.function_params + + +def test_empty_status_response_is_retryable(mig): + with pytest.raises(trb.TaskRetry): + mig.report_migration_status(_task(), None) + + +# -- migration recovery gate ------------------------------------------------ + +def test_recovery_gate_passes_a_whole_cluster(mig): + task = _task(**{mig.MIGRATION_WAIT_UNAVAILABLE_KEY: ["node:n1"]}) + assert mig.require_recovery_progress(task, []) is None + assert mig.MIGRATION_WAIT_UNAVAILABLE_KEY not in task.function_params + + +def test_recovery_gate_defers_while_nothing_changes(mig): + """Retrying against an unchanged outage would burn the budget and + terminate the migration before the cluster came back.""" + task = _task(**{mig.MIGRATION_WAIT_UNAVAILABLE_KEY: ["node:n1"]}) + with pytest.raises(trb.TaskDefer): + mig.require_recovery_progress(task, ["node:n1"]) + + +def test_recovery_gate_releases_on_a_recovery_event(mig): + task = _task(**{mig.MIGRATION_WAIT_UNAVAILABLE_KEY: ["node:n1", "dev:d1"]}) + assert mig.require_recovery_progress(task, ["dev:d1"]) is None + assert task.function_params[mig.MIGRATION_WAIT_UNAVAILABLE_KEY] == ["dev:d1"] + + +def test_recovery_gate_defers_on_a_first_outage(mig): + with pytest.raises(trb.TaskDefer): + mig.require_recovery_progress(_task(), ["node:n1"]) + + +# -- migration-family eligibility ------------------------------------------- + +def test_sibling_gate_blocks_a_task_that_has_not_started(mig): + mig.tasks_controller.get_active_node_mig_task.return_value = "other-task" + assert mig.no_sibling_migration(_task(distr_name="distr-1")) is False + + +def test_sibling_gate_releases_a_task_already_migrating(mig): + """Once its own migration is running the task IS the sibling others wait + on — gating it here would stall it forever.""" + mig.tasks_controller.get_active_node_mig_task.return_value = "other-task" + task = _task(distr_name="distr-1", migration={"name": "distr-1"}) + assert mig.no_sibling_migration(task) is True + + +# -- failed migration ------------------------------------------------------- + +@pytest.fixture +def failed_migration(monkeypatch): + import simplyblock_core.services.tasks_runner_failed_migration as runner + monkeypatch.setattr(runner, "db", MagicMock()) + monkeypatch.setattr(runner, "tasks_controller", MagicMock()) + monkeypatch.setattr(runner, "device_controller", MagicMock()) + return runner + + +def test_failed_migration_tags_the_device_once_finished(failed_migration): + failed_migration.tasks_controller.get_failed_device_mig_task.return_value = False + task = _task(distr_name="distr-1", migration={"name": "distr-1"}) + task.device_id = "dev-1" + + failed_migration.SPEC.on_finish(task) + + failed_migration.device_controller.device_set_failed_and_migrated.assert_called_once_with("dev-1") + + +def test_failed_migration_leaves_the_device_alone_if_never_started(failed_migration): + failed_migration.tasks_controller.get_failed_device_mig_task.return_value = False + task = _task(distr_name="distr-1") + task.device_id = "dev-1" + + failed_migration.SPEC.on_finish(task) + + failed_migration.device_controller.device_set_failed_and_migrated.assert_not_called() + + +def test_failed_migration_waits_for_the_last_task_on_the_device(failed_migration): + failed_migration.tasks_controller.get_failed_device_mig_task.return_value = "other-task" + task = _task(distr_name="distr-1", migration={"name": "distr-1"}) + task.device_id = "dev-1" + + failed_migration.SPEC.on_finish(task) + + failed_migration.device_controller.device_set_failed_and_migrated.assert_not_called() diff --git a/tests/unit/tasks/test_task_runner_base.py b/tests/unit/tasks/test_task_runner_base.py index a8563e4816..13df330d0f 100644 --- a/tests/unit/tasks/test_task_runner_base.py +++ b/tests/unit/tasks/test_task_runner_base.py @@ -615,3 +615,37 @@ def test_failing_cycle_hook_does_not_stop_the_loop(monkeypatch): runner = _runner(MagicMock(), on_cycle=MagicMock(side_effect=RuntimeError("watchdog boom"))) with pytest.raises(_StopLoop): # reached the sleep, i.e. the cycle completed runner.run() + + +# -- in-progress polling ---------------------------------------------------- + +def test_progress_keeps_the_task_running(monkeypatch): + """A polled long-running operation must not be suspended between polls: + the migration family gates mutual exclusion on a sibling being RUNNING.""" + task = _task(retry=2) + store = _wire(monkeypatch, task) + + def handler(_task): + raise trb.TaskProgress("Status: in_progress, progress:42") + + runner = _runner(handler) + runner._process(task, MagicMock()) + + assert store.row().status == JobSchedule.STATUS_RUNNING + assert store.row().retry == 2 + assert store.row().function_result == "Status: in_progress, progress:42" + assert "task-1" not in runner._next_attempt + + +def test_progress_does_not_resurrect_a_concurrently_canceled_task(monkeypatch): + task = _task() + store = _wire(monkeypatch, task) + + def handler(_task): + store.concurrently(canceled=True, status=JobSchedule.STATUS_DONE) + raise trb.TaskProgress("still going") + + _runner(handler)._process(task, MagicMock()) + + assert store.row().canceled is True + assert store.row().status == JobSchedule.STATUS_DONE diff --git a/tests/unit/test_node_removal.py b/tests/unit/test_node_removal.py index 3a0b78840a..af96ba3a9c 100644 --- a/tests/unit/test_node_removal.py +++ b/tests/unit/test_node_removal.py @@ -478,28 +478,36 @@ class TestShrinkStatusDoesNotDeadlockRemoval(unittest.TestCase): def _run_gate(self, cluster_status): """Drive the runner's cluster-status gate; True == it proceeded.""" from simplyblock_core.services import tasks_runner_failed_migration as runner + from simplyblock_core.services import migration_task_common as mig + from simplyblock_core.services.task_runner_base import TaskRetry from simplyblock_core.models.job_schedule import JobSchedule task = MagicMock(spec=JobSchedule) task.node_id, task.cluster_id, task.retry = "n1", "cl-1", 0 task.status = JobSchedule.STATUS_RUNNING + task.function_params = {} - cluster = MagicMock(spec=Cluster) + # A real Cluster, not a mock: the gate goes through Cluster's own + # status predicates, which a spec'd mock would answer truthily for + # every status and so pass the gate it is meant to test. + cluster = Cluster() cluster.status = cluster_status db = MagicMock() db.get_cluster_by_id.return_value = cluster db.get_storage_node_by_id.return_value = _node("n1", n_devices=1, with_jm=False) - with patch.object(runner, "db", db): + with patch.object(runner, "db", db), patch.object(mig, "db", db): try: runner.task_runner(task) + except TaskRetry as e: + # The gate refuses by raising; the driver, not the handler, is + # what turns that into STATUS_SUSPENDED. Any other TaskRetry + # comes from the real device/RPC work past the gate, which this + # test deliberately does not mock. + return "cluster is not active" not in str(e) except Exception: - # Admitted by the gate, then reached real device/DB work this - # test deliberately does not mock. The gate's decision is - # already recorded on task.status by that point, and it is the - # only thing under test here. pass - return task.status != JobSchedule.STATUS_SUSPENDED + return True def test_failed_migration_runner_admits_in_shrink(self): self.assertTrue( From 4ade335cd7328433dc1e7974fb7e1ef27f98ad10 Mon Sep 17 00:00:00 2001 From: Max Schettler Date: Thu, 6 Aug 2026 02:12:36 +0200 Subject: [PATCH 23/25] Migrate node removal and port allow runners onto the task runner MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both are mechanical translations of their suspend points into the handler vocabulary, and neither had a retry counter to hand over — every gate in them deferred. Node removal's "incomplete, retry later" pass becomes TaskProgress. It is progress, not failure: the removal waits on device failure-migration that can take hours, and treating each poll as a retry would have earned it an exponential backoff it never had. Port allow keeps its documented opt-out — no IN_ACTIVATION eligibility gate, because activation needs exactly these ports open. Two of its paths returned bare, leaving the task untouched for the loop to revisit; under the driver a bare return means success, so both are now explicit defers. Its abort-on-half-open-hublvol path keeps the abort side effect in the handler and raises TaskAbort for the outcome. Adds an import-smoke over all thirteen migrated runners that also asserts no two of them claim the same task function name — the lease guards against a second host, not against a second spec. --- .../services/tasks_runner_node_removal.py | 92 ++------ .../services/tasks_runner_port_allow.py | 208 +++++------------- tests/integration/test_port_allow_recovery.py | 97 ++++---- tests/unit/tasks/test_retry_ceiling.py | 3 +- tests/unit/tasks/test_runner_specs.py | 43 ++++ 5 files changed, 163 insertions(+), 280 deletions(-) diff --git a/simplyblock_core/services/tasks_runner_node_removal.py b/simplyblock_core/services/tasks_runner_node_removal.py index 412cb4dcaa..1df89f78c9 100644 --- a/simplyblock_core/services/tasks_runner_node_removal.py +++ b/simplyblock_core/services/tasks_runner_node_removal.py @@ -1,11 +1,12 @@ # coding=utf-8 -import time - - from simplyblock_core import db_controller, storage_node_ops, utils, constants -from simplyblock_core.controllers import tasks_controller from simplyblock_core.models.job_schedule import JobSchedule from simplyblock_core.models.cluster import Cluster +from simplyblock_core.services.task_runner_base import ( + RunnerSpec, + TaskProgress, + serve, +) logger = utils.get_logger(__name__) @@ -19,76 +20,29 @@ def process_task(task): node_removal_orchestrate is idempotent and resumable: it returns True only when the node is fully REMOVED, and False to mean "incomplete, retry later" - (most commonly: device failure-migration still in progress). On False we - suspend the task so the outer loop revisits it on the next tick instead of - busy-spinning here for what can be hours. + (most commonly: device failure-migration still in progress, which can take + hours). Incomplete is progress, not failure — it consumes no retry, and the + task stays RUNNING so the next tick picks it straight back up. """ - if task.canceled: - task.function_result = "canceled" - task.status = JobSchedule.STATUS_DONE - task.write_to_db(db.kv_store) - return True - - cluster = db.get_cluster_by_id(task.cluster_id) - if cluster.status == Cluster.STATUS_IN_ACTIVATION: - task.function_result = "cluster is in_activation, waiting" - task.status = JobSchedule.STATUS_SUSPENDED - task.write_to_db(db.kv_store) - return False + force_remove = bool(task.function_params.get("force_remove", False)) + if not storage_node_ops.node_removal_orchestrate(task.node_id, force_remove=force_remove): + raise TaskProgress("removal in progress, retrying") - if task.status != JobSchedule.STATUS_RUNNING: - task.status = JobSchedule.STATUS_RUNNING - task.write_to_db(db.kv_store) + task.function_result = "Node removed" - force_remove = bool(task.function_params.get("force_remove", False)) - try: - done = storage_node_ops.node_removal_orchestrate(task.node_id, force_remove=force_remove) - except Exception as e: - logger.error(f"Node-removal task {task.uuid} raised: {e}") - logger.exception(e) - task.function_result = f"error: {e}" - task.retry += 1 - task.status = JobSchedule.STATUS_SUSPENDED - task.write_to_db(db.kv_store) - return False - if done: - task.function_result = "Node removed" - task.status = JobSchedule.STATUS_DONE - task.write_to_db(db.kv_store) - return True +SPEC = RunnerSpec( + name="tasks-runner-node-removal", + function_names=[JobSchedule.FN_NODE_REMOVAL], + handler=process_task, + is_eligible=lambda task, cluster: cluster.status != Cluster.STATUS_IN_ACTIVATION, + interval=constants.TASK_EXEC_INTERVAL_SEC, +) - # Incomplete: a phase asked us to retry (typically waiting on migration). - task.function_result = "removal in progress, retrying" - task.retry += 1 - task.status = JobSchedule.STATUS_SUSPENDED - task.write_to_db(db.kv_store) - return False +def main(): + serve(SPEC) -logger.info("Starting Tasks runner node removal...") -while True: - time.sleep(constants.TASK_EXEC_INTERVAL_SEC) - clusters = db.get_clusters() - if not clusters: - logger.error("No clusters found!") - continue - for cl in clusters: - tasks = db.get_job_tasks(cl.get_id(), reverse=False) - for task in tasks: - if task.function_name != JobSchedule.FN_NODE_REMOVAL: - continue - if task.status == JobSchedule.STATUS_DONE: - continue - # get a fresh object: cancel/other writers may have changed it - task = db.get_task_by_id(task.uuid) - # Lease gate: skip a task another live runner host owns. - if not tasks_controller.claim_task(task): - logger.info(f"Node-removal task {task.uuid} owned by another runner host; skipping") - continue - try: - process_task(task) - except Exception as e: - logger.error(f"Node-removal task {task.uuid} processing crashed: {e}") - logger.exception(e) +if __name__ == "__main__": + main() diff --git a/simplyblock_core/services/tasks_runner_port_allow.py b/simplyblock_core/services/tasks_runner_port_allow.py index 6868dcbe99..16bf4abaa6 100644 --- a/simplyblock_core/services/tasks_runner_port_allow.py +++ b/simplyblock_core/services/tasks_runner_port_allow.py @@ -12,6 +12,12 @@ from simplyblock_core.models.storage_node import StorageNode from simplyblock_core.models.nvme_device import NVMeDevice from simplyblock_core.models.lvol_model import LVol +from simplyblock_core.services.task_runner_base import ( + RunnerSpec, + TaskAbort, + TaskDefer, + serve, +) logger = utils.get_logger(__name__) @@ -566,30 +572,13 @@ def _abort_recovering_node(node, reason): def exec_port_allow_task(task): - # get new task object because it could be changed from cancel task - task = db.get_task_by_id(task.uuid) - - if task.canceled: - task.function_result = "canceled" - task.status = JobSchedule.STATUS_DONE - task.write_to_db(db.kv_store) - return - try: node = db.get_storage_node_by_id(task.node_id) except KeyError: - task.function_result = "node not found" - task.status = JobSchedule.STATUS_DONE - task.write_to_db(db.kv_store) - return + raise TaskAbort("node not found") if node.status not in [StorageNode.STATUS_DOWN, StorageNode.STATUS_ONLINE]: - msg = f"Node is {node.status}, retry task" - logger.info(msg) - task.function_result = msg - task.status = JobSchedule.STATUS_SUSPENDED - task.write_to_db(db.kv_store) - return + raise TaskDefer(f"Node is {node.status}, retry task") # check node ping ping_check = health_controller._check_node_ping(node.mgmt_ip) @@ -600,12 +589,7 @@ def exec_port_allow_task(task): logger.info(f"Check 2: ping mgmt ip {node.mgmt_ip} ... {ping_check}") if not ping_check: - msg = "Node ping is false, retry task" - logger.info(msg) - task.function_result = msg - task.status = JobSchedule.STATUS_SUSPENDED - task.write_to_db(db.kv_store) - return + raise TaskDefer("Node ping is false, retry task") # Data-NIC gate: mgmt reachability alone is not recovery — after a # partial partition the mgmt plane often returns first while the @@ -629,24 +613,14 @@ def exec_port_allow_task(task): logger.info(f"Check: ping data nic {data_nic.ip4_address} ... {data_ping}") data_results.append(data_ping) if data_results and not any(r is True for r in data_results): - msg = "Node data NIC not confirmed reachable, retry task" - logger.info(msg) - task.function_result = msg - task.status = JobSchedule.STATUS_SUSPENDED - task.write_to_db(db.kv_store) - return + raise TaskDefer("Node data NIC not confirmed reachable, retry task") logger.info("connect to remote devices") # connect to remote devs try: remote_devices = storage_node_ops._connect_to_remote_devs(node, reattach=False) if not remote_devices: - msg = "Node unable to connect to remote devs, retry task" - logger.info(msg) - task.function_result = msg - task.status = JobSchedule.STATUS_SUSPENDED - task.write_to_db(db.kv_store) - return + raise TaskDefer("Node unable to connect to remote devs, retry task") else: # Re-read fresh before writing to avoid overwriting concurrent changes node = db.get_storage_node_by_id(task.node_id) @@ -656,12 +630,7 @@ def exec_port_allow_task(task): logger.info("connect to remote JM devices") remote_jm_devices = storage_node_ops._connect_to_remote_jm_devs(node) if not remote_jm_devices or len(remote_jm_devices) < 2: - msg = "Node unable to connect to remote JMs, retry task" - logger.info(msg) - task.function_result = msg - task.status = JobSchedule.STATUS_SUSPENDED - task.write_to_db(db.kv_store) - return + raise TaskDefer("Node unable to connect to remote JMs, retry task") else: # Re-read fresh before writing to avoid overwriting concurrent changes node = db.get_storage_node_by_id(task.node_id) @@ -669,14 +638,11 @@ def exec_port_allow_task(task): node.write_to_db() + except TaskDefer: + raise except Exception as e: logger.error(e) - msg = "Error when connect to remote devs, retry task" - logger.info(msg) - task.function_result = msg - task.status = JobSchedule.STATUS_SUSPENDED - task.write_to_db(db.kv_store) - return + raise TaskDefer("Error when connect to remote devs, retry task") # After a network outage every distrib in the cluster is working from a # stale device view: the recovering node's distribs have stale REMOTE @@ -707,12 +673,7 @@ def exec_port_allow_task(task): f"port allow on {node.get_id()}") if not device_controller.device_set_online( dev.get_id(), cause=device_controller.CAUSE_NODE_RECOVERY): - msg = f"Device {dev.get_id()} re-admit refused, retry task" - logger.warning(msg) - task.function_result = msg - task.status = JobSchedule.STATUS_SUSPENDED - task.write_to_db(db.kv_store) - return + raise TaskDefer(f"Device {dev.get_id()} re-admit refused, retry task") # Positive confirmation gate: every device-status event must be applied by # ALL of its target distribs before we touch hublvols / leadership / the @@ -722,12 +683,6 @@ def exec_port_allow_task(task): # held a stale device view, which is the placement/read-failure class this # whole sequence exists to prevent. On any device whose event is not # confirmed, suspend and retry — do NOT proceed. - def _fail(msg): - logger.warning(msg) - task.function_result = msg - task.status = JobSchedule.STATUS_SUSPENDED - task.write_to_db(db.kv_store) - logger.info("Broadcasting local device status of recovering node to all nodes") node = db.get_storage_node_by_id(task.node_id) for dev in node.nvme_devices: @@ -736,12 +691,11 @@ def _fail(msg): try: ok = distr_controller.send_dev_status_event(dev, dev.status) except Exception as e: - _fail(f"Local device status broadcast for {dev.get_id()} failed: {e}, retry task") - return + raise TaskDefer(f"Local device status broadcast for {dev.get_id()} " + f"failed: {e}, retry task") if not ok: - _fail(f"Local device status for {dev.get_id()} not applied by all " - f"distribs, retry task") - return + raise TaskDefer(f"Local device status for {dev.get_id()} " + f"not applied by all distribs, retry task") logger.info("Sending other nodes' device status events to recovering node") for cluster_node in db.get_storage_nodes_by_cluster_id(task.cluster_id): @@ -753,14 +707,14 @@ def _fail(msg): try: ok = distr_controller.send_dev_status_event( dev, dev.status, target_node=node) + except TaskDefer: + raise except Exception as e: - _fail(f"Device status event for {dev.get_id()} to recovering node " - f"failed: {e}, retry task") - return + raise TaskDefer(f"Device status event for {dev.get_id()} to " + f"recovering node failed: {e}, retry task") if not ok: - _fail(f"Device status for {dev.get_id()} not applied by recovering " - f"node's distribs, retry task") - return + raise TaskDefer(f"Device status for {dev.get_id()} " + f"not applied by recovering node's distribs, retry task") logger.info("All device-status events confirmed applied by distribs") @@ -774,24 +728,14 @@ def _fail(msg): node = db.get_storage_node_by_id(task.node_id) own_hublvols_ok, own_msg = _reconnect_own_sec_tert_hublvols(node) if not own_hublvols_ok: - msg = f"Own (outbound) hublvol reconnect failed: {own_msg}, retry task" - logger.warning(msg) - task.function_result = msg - task.status = JobSchedule.STATUS_SUSPENDED - task.write_to_db(db.kv_store) - return + raise TaskDefer(f"Own (outbound) hublvol reconnect failed: {own_msg}, retry task") # INBOUND: for lvstores where this node is the secondary, re-expose its # hub and ensure the tertiary's redirect path to it is connected (no # leadership action here — that stays in the failback step below). inbound_ok, inbound_msg = _reconnect_inbound_hublvols(node) if not inbound_ok: - msg = f"Inbound hublvol reconnect failed: {inbound_msg}, retry task" - logger.warning(msg) - task.function_result = msg - task.status = JobSchedule.STATUS_SUSPENDED - task.write_to_db(db.kv_store) - return + raise TaskDefer(f"Inbound hublvol reconnect failed: {inbound_msg}, retry task") snode = db.get_storage_node_by_id(node.get_id()) sec_ids = [] @@ -821,17 +765,12 @@ def _fail(msg): snode.jm_vuid) except Exception as e: logger.error(e) - return + raise TaskDefer(f"JC compression check on peer failed: {e}, retry task") if node.lvstore_status == "ready": lvstore_check = health_controller._check_node_lvstore(node.lvstore_stack, node, auto_fix=True) if not lvstore_check: - msg = "Node LVolStore check fail, retry later" - logger.warning(msg) - task.function_result = msg - task.status = JobSchedule.STATUS_SUSPENDED - task.write_to_db(db.kv_store) - return + raise TaskDefer("Node LVolStore check fail, retry later") sec_ids = [] if node.secondary_node_id: @@ -846,12 +785,7 @@ def _fail(msg): # primary-local recovery step, not a peer reconnect. primary_hublvol_check = health_controller._check_node_hublvol(node) if not primary_hublvol_check: - msg = "Node hublvol check fail, retry later" - logger.warning(msg) - task.function_result = msg - task.status = JobSchedule.STATUS_SUSPENDED - task.write_to_db(db.kv_store) - return + raise TaskDefer("Node hublvol check fail, retry later") # Peer hublvol gate: for each ONLINE secondary/tertiary, drive # ``connect_to_hublvol`` (which re-attaches the bdev_nvme @@ -883,15 +817,8 @@ def _fail(msg): f"after {_HUBLVOL_MAX_ATTEMPTS} attempts: " + ", ".join(p[:8] for p in failing_peers)) _abort_recovering_node(node, reason) - task.function_result = ( + raise TaskAbort( f"Aborted recovering node {node.get_id()[:8]}: {reason}") - task.status = JobSchedule.STATUS_DONE - task.write_to_db(db.kv_store) - return - - if task.status != JobSchedule.STATUS_RUNNING: - task.status = JobSchedule.STATUS_RUNNING - task.write_to_db(db.kv_store) try: # wait for lvol sync delete @@ -903,9 +830,11 @@ def _fail(msg): port_number = task.function_params["port_number"] + except TaskDefer: + raise except Exception as e: logger.error(e) - return + raise TaskDefer(f"Waiting for lvol sync delete failed: {e}, retry task") # --- Leadership failback, BEFORE the port is unblocked -------------- # @@ -954,12 +883,8 @@ def _fail(msg): leadership_read_failed = f"{peer.get_id()[:8]}: {e}" if current_leader is None and leadership_read_failed: - msg = f"Leadership read failed on peer {leadership_read_failed}, retry task" - logger.warning(msg) - task.function_result = msg - task.status = JobSchedule.STATUS_SUSPENDED - task.write_to_db(db.kv_store) - return + raise TaskDefer( + f"Leadership read failed on peer {leadership_read_failed}, retry task") if current_leader is not None: failback_ok, failback_msg = _failback_leadership_to_primary( @@ -974,12 +899,7 @@ def _fail(msg): failback_ok, failback_msg = True, "" if not failback_ok: - msg = f"Leadership failback incomplete: {failback_msg}, retry task" - logger.warning(msg) - task.function_result = msg - task.status = JobSchedule.STATUS_SUSPENDED - task.write_to_db(db.kv_store) - return + raise TaskDefer(f"Leadership failback incomplete: {failback_msg}, retry task") # Unblock ALL of the node's LVS subsystem ports (own primary + every # follower LVS), not just the single port_number the task was created @@ -1061,41 +981,27 @@ def _fail(msg): logger.error(f"Device re-admit after port allow failed: {e}") task.function_result = f"Port {port_number} allowed on node" - task.status = JobSchedule.STATUS_DONE - task.write_to_db(db.kv_store) -def _main(): - logger.info("Starting Tasks runner...") - while True: - clusters = db.get_clusters() - if not clusters: - logger.error("No clusters found!") - else: - for cl in clusters: - # Deliberately NOT skipped while the cluster is IN_ACTIVATION: - # port_allow is the final step of a node's recovery, and - # activation NEEDS those ports open (2026-07-16 full-fleet - # reboot: a task suspended seconds before activation started - # froze at 0/8 retries for the entire 30-minute activation, - # while the activation's hublvol attaches to that node failed - # against its still-blocked port). The task's own gates (node - # status, mgmt/data-NIC pings, verified-open hublvols) decide - # whether a retry can proceed; a retry that still can't just - # re-suspends. - tasks = db.get_job_tasks(cl.get_id(), reverse=False) - for task in tasks: - if task.function_name == JobSchedule.FN_PORT_ALLOW: - if task.status != JobSchedule.STATUS_DONE: - # Lease gate: skip a task another live runner host owns. - if not tasks_controller.claim_task(task): - logger.info(f"Port-allow task {task.uuid} owned by another runner host; skipping") - continue - with tasks_controller.task_lease_heartbeat(task): - exec_port_allow_task(task) - - time.sleep(5) +SPEC = RunnerSpec( + name="tasks-runner-port-allow", + function_names=[JobSchedule.FN_PORT_ALLOW], + handler=exec_port_allow_task, + # No IN_ACTIVATION gate, deliberately, unlike every other runner: + # port_allow is the final step of a node's recovery, and activation NEEDS + # those ports open (2026-07-16 full-fleet reboot: a task suspended seconds + # before activation started froze for the entire 30-minute activation, + # while the activation's hublvol attaches to that node failed against its + # still-blocked port). The task's own gates — node status, mgmt/data-NIC + # pings, verified-open hublvols — decide whether a retry can proceed; one + # that still can't just defers again. + interval=5, +) + + +def main(): + serve(SPEC) if __name__ == "__main__": - _main() + main() diff --git a/tests/integration/test_port_allow_recovery.py b/tests/integration/test_port_allow_recovery.py index da53989864..01a66e46c8 100644 --- a/tests/integration/test_port_allow_recovery.py +++ b/tests/integration/test_port_allow_recovery.py @@ -63,6 +63,7 @@ from simplyblock_core.models.storage_node import StorageNode from simplyblock_core.models.nvme_device import NVMeDevice from simplyblock_core.models.lvol_model import LVol +from simplyblock_core.services.task_runner_base import TaskAbort, TaskDefer def _make_lvol(uuid, status=None): @@ -327,7 +328,6 @@ def _run(self): def test_full_cluster_map_push_is_gone(self): self._run() self.cluster_map_push.assert_not_called() - self.assertEqual(self.task.status, JobSchedule.STATUS_DONE) def test_readmit_uses_node_recovery_cause_and_precedes_unblock(self): self._run() @@ -387,9 +387,8 @@ def test_readmit_refused_suspends_task(self): "simplyblock_core.services.tasks_runner_port_allow." "device_controller.device_set_online", return_value=False, - ): + ), self.assertRaises(TaskDefer): exec_port_allow_task(self.task) - self.assertEqual(self.task.status, JobSchedule.STATUS_SUSPENDED) self.assertEqual(self._node_allows(), [], "no firewall allow when the pre-unblock re-admit " "was refused") @@ -403,9 +402,8 @@ def test_broadcast_failure_suspends_task(self): "simplyblock_core.services.tasks_runner_port_allow." "distr_controller.send_dev_status_event", side_effect=Exception("distrib send failed"), - ): + ), self.assertRaises(TaskDefer): exec_port_allow_task(self.task) - self.assertEqual(self.task.status, JobSchedule.STATUS_SUSPENDED) self.assertEqual(self._node_allows(), [], "no firewall allow when the device-status refresh " "failed") @@ -443,21 +441,20 @@ def _node_allows(self): ] def test_data_nic_down_suspends(self): - self._run_with_data_ping(False) - self.assertEqual(self.task.status, JobSchedule.STATUS_SUSPENDED) + with self.assertRaises(TaskDefer): + self._run_with_data_ping(False) self.assertEqual(self._node_allows(), [], "the port must not open while the node's data NIC is down") def test_data_nic_inconclusive_suspends(self): - self._run_with_data_ping(None) - self.assertEqual(self.task.status, JobSchedule.STATUS_SUSPENDED) + with self.assertRaises(TaskDefer): + self._run_with_data_ping(None) self.assertEqual(self._node_allows(), [], "an inconclusive data-NIC ping must not allow the port — " "recovery requires positive confirmation") def test_data_nic_up_proceeds(self): self._run_with_data_ping(True) - self.assertEqual(self.task.status, JobSchedule.STATUS_DONE) self.assertEqual(len(self._node_allows()), 1) @@ -560,7 +557,6 @@ def test_no_failback_when_no_peer_leader(self): jc_calls = [c for c in self.calls if c[0] == "jc_disable_replication"] self.assertEqual(jc_calls, [], "no quiesce without an acting leader") self.assertEqual(len(self._node_allows()), 1) - self.assertEqual(self.task.status, JobSchedule.STATUS_DONE) def test_zero_leader_left_alone(self): # Nobody holds leadership (no-abort outage aftermath): the runner @@ -576,7 +572,6 @@ def test_zero_leader_left_alone(self): "a zero-leader LVS is left for the primary's " "promotion-on-first-IO — no CP-side healing") self.assertEqual(len(self._node_allows()), 1) - self.assertEqual(self.task.status, JobSchedule.STATUS_DONE) def test_quiesce_loop_retries_locally_then_succeeds(self): # jc_disable_replication False = active replication -> re-quiesce @@ -597,7 +592,6 @@ def _jc(vuid): demotes = [c for c in self.calls if c[0] == "bdev_lvol_set_leader" and c[1] == "sec"] self.assertEqual(len(demotes), 1) - self.assertEqual(self.task.status, JobSchedule.STATUS_DONE) def test_quiesce_loop_exhaustion_suspends(self): from simplyblock_core.services.tasks_runner_port_allow import ( @@ -609,11 +603,11 @@ def _jc(vuid): return False self.sec_rpc.jc_disable_replication.side_effect = _jc - self._run() + with self.assertRaises(TaskDefer): + self._run() jc_calls = [c for c in self.calls if c[0] == "jc_disable_replication"] self.assertEqual(len(jc_calls), _REPL_SUSPEND_MAX_ATTEMPTS, "the quiesce+disable loop is bounded locally") - self.assertEqual(self.task.status, JobSchedule.STATUS_SUSPENDED) demotes = [c for c in self.calls if c[0] == "bdev_lvol_set_leader"] self.assertEqual(demotes, [], "no demote against active journal replication") @@ -622,8 +616,8 @@ def _jc(vuid): def test_jc_disable_raise_suspends(self): self.sec_rpc.jc_disable_replication.side_effect = Exception("jc rpc timeout") - self._run() - self.assertEqual(self.task.status, JobSchedule.STATUS_SUSPENDED) + with self.assertRaises(TaskDefer): + self._run() demotes = [c for c in self.calls if c[0] == "bdev_lvol_set_leader"] self.assertEqual(demotes, []) self.assertEqual(self._node_allows(), []) @@ -638,9 +632,8 @@ def test_down_acting_leader_hublvol_is_still_a_hard_gate(self): "simplyblock_core.services.tasks_runner_port_allow." "_verify_or_reconnect_peer_hublvol", return_value=False, - ): + ), self.assertRaises(TaskDefer): self._run() - self.assertEqual(self.task.status, JobSchedule.STATUS_SUSPENDED) demotes = [c for c in self.calls if c[0] == "bdev_lvol_set_leader"] self.assertEqual(demotes, [], "no demote while the acting leader's hublvol to the " @@ -722,7 +715,6 @@ def test_demote_lands_on_tertiary_only_and_is_plain(self): if c[0] == "bdev_distrib_force_to_non_leader"] self.assertEqual(forces, [], "no force_to_non_leader in the minimal failback") - self.assertEqual(self.task.status, JobSchedule.STATUS_DONE) def test_quiesce_verify_demote_order_with_tertiary_leader(self): self._run() @@ -1021,7 +1013,7 @@ def test_strict_verify_exhausts_retries_aborts_recovering_node(self): ) as reconnect_mock, patch( "simplyblock_core.services.tasks_runner_port_allow._abort_recovering_node", side_effect=lambda n, r: self.calls.append(("abort_recovering_node", n.uuid, r)), - ) as abort_mock: + ) as abort_mock, self.assertRaises(TaskAbort): exec_port_allow_task(self.task) abort_mock.assert_called_once() @@ -1037,8 +1029,8 @@ def test_strict_verify_exhausts_retries_aborts_recovering_node(self): if c[0] == "firewall_set_port" and c[3] == "allow"] self.assertEqual(allow_calls, []) - # Task ended in DONE (not SUSPENDED — the retries already ran). - self.assertEqual(self.task.status, JobSchedule.STATUS_DONE) + # TaskAbort, not TaskDefer: the retries already ran, so the driver + # must finish the task rather than schedule another attempt. # 5 reconnect attempts were issued (one per retry iteration). self.assertEqual(reconnect_mock.call_count, 5) @@ -1141,22 +1133,23 @@ def test_abort_helper_present_and_used(self): self.assertIn("_abort_recovering_node(node, reason)", self.src) def test_no_port_allowed_after_abort(self): - # Source-level invariant: when the abort path runs the task returns - # without falling through to firewall_set_port/port_allowed. - # We assert that abort sets task status DONE and the function - # returns before the firewall/port_allowed lines. + # Source-level invariant: when the abort path runs, the handler + # leaves via TaskAbort without falling through to + # firewall_set_port/port_allowed. The raise is what makes the + # fall-through impossible — it replaced an explicit STATUS_DONE + + # return when the runner moved onto the task runner base. i_abort = self.src.find("_abort_recovering_node(node, reason)") - i_done = self.src.find("STATUS_DONE", i_abort) - i_return = self.src.find("return", i_done) + i_raise = self.src.find("raise TaskAbort", i_abort) # The invariant is about the port_allowed occurrence in - # exec_port_allow_task AFTER the abort path's return. + # exec_port_allow_task AFTER the abort path's raise. i_allow_event = self.src.find("tcp_ports_events.port_allowed", i_abort) self.assertGreater(i_abort, 0) - self.assertGreater(i_done, i_abort) - self.assertGreater(i_return, i_done) - self.assertGreater(i_allow_event, i_return, + self.assertGreater(i_raise, i_abort, + "the abort path must terminate the handler with " + "TaskAbort, not fall through") + self.assertGreater(i_allow_event, i_raise, "tcp_ports_events.port_allowed must appear after " - "the abort path's return so it cannot fire on the " + "the abort path's raise so it cannot fire on the " "abort code path") @@ -1257,9 +1250,8 @@ def test_pre_unblock_refusal_suspends_task(self): "simplyblock_core.services.tasks_runner_port_allow." "device_controller.device_set_online", return_value=False, - ): + ), self.assertRaises(TaskDefer): self._run() - self.assertEqual(self.task.status, JobSchedule.STATUS_SUSPENDED) node_allows = [ c for c in self.calls if c[0] == "firewall_set_port" and c[1] == self.node.uuid @@ -1291,7 +1283,6 @@ def _refuse_epilogue(dev_id, *a, **kw): self.assertTrue( any("refused" in line for line in logs.output), "a refused re-admit must be logged, never silent") - self.assertEqual(self.task.status, JobSchedule.STATUS_DONE) class _SecRoleReconnectBase(_BasePortAllowTest): @@ -1405,21 +1396,21 @@ def test_tertiary_path_failure_is_a_hard_gate(self): self.node_rpc.subsystem_list.return_value = [{"nqn": "nqn-p-hub"}] self.node_rpc.subsystem_get.return_value = {"nqn": "nqn-p-hub"} self.tert.add_hublvol_failover_path = MagicMock(return_value=False) - self._run_with_verify(lambda *a: True) + with self.assertRaises(TaskDefer): + self._run_with_verify(lambda *a: True) node_allows = [ c for c in self.calls if c[0] == "firewall_set_port" and c[1] == self.node.uuid and c[3] == "allow" ] self.assertEqual(node_allows, [], "the port must not open with a broken tertiary redirect") - self.assertEqual(self.task.status, JobSchedule.STATUS_SUSPENDED) def test_outbound_failure_suspends_task(self): def _fail_only_own(peer, primary): # Fail the node->prim direction; pass the peer-gate direction. return not (peer is self.node and primary is self.prim) - self._run_with_verify(_fail_only_own) - self.assertEqual(self.task.status, JobSchedule.STATUS_SUSPENDED) + with self.assertRaises(TaskDefer): + self._run_with_verify(_fail_only_own) node_allows = [ c for c in self.calls if c[0] == "firewall_set_port" and c[1] == self.node.uuid and c[3] == "allow" @@ -1439,7 +1430,6 @@ def test_offline_primary_outbound_skipped(self): self.assertEqual(own_calls, [], "no OUTBOUND reconnect toward a non-ONLINE primary — " "its own recovery path re-drives that leg") - self.assertEqual(self.task.status, JobSchedule.STATUS_DONE) def test_follower_recovery_is_hublvol_wiring_only(self): # Scenario (iv): the recovering node is a plain follower of an @@ -1462,7 +1452,6 @@ def test_follower_recovery_is_hublvol_wiring_only(self): "triggers no leadership action at all") ana_mock.assert_not_called() self.assertEqual(len(self._node_allows()), 1) - self.assertEqual(self.task.status, JobSchedule.STATUS_DONE) def test_no_ana_promotion_when_primary_online(self): # The deferred post-unblock ANA promotion is gated on the primary @@ -1478,7 +1467,6 @@ def test_no_ana_promotion_when_primary_online(self): ) as ana_mock: self._run_with_verify(lambda *a: True) ana_mock.assert_not_called() - self.assertEqual(self.task.status, JobSchedule.STATUS_DONE) class TestStaleLeaderConvergence(_SecRoleReconnectBase): @@ -1538,7 +1526,6 @@ def test_stale_claim_not_converged_by_runner(self): if c[0] == "bdev_distrib_force_to_non_leader" and c[1] == "node"] self.assertEqual(forces, []) self.assertEqual(len(self._node_allows()), 1) - self.assertEqual(self.task.status, JobSchedule.STATUS_DONE) def test_no_demote_when_primary_not_leading(self): # The node is then the legitimate acting leader; demoting it would @@ -1547,13 +1534,11 @@ def test_no_demote_when_primary_not_leading(self): self.prim_rpc.bdev_lvol_get_lvstores.return_value = [{"lvs leadership": False}] self._run() self.assertEqual(self._node_demotes(), []) - self.assertEqual(self.task.status, JobSchedule.STATUS_DONE) def test_no_action_without_stale_claim(self): self.lvs_leadership_on_node["LVS_P"] = False self._run() self.assertEqual(self._node_demotes(), []) - self.assertEqual(self.task.status, JobSchedule.STATUS_DONE) class TestNoFollowerPortFencing(_SecRoleReconnectBase): @@ -1583,13 +1568,12 @@ def test_down_follower_never_fenced(self): "at first contact — the fencing was removed on 2026-07-07") self.assertNotIn("fenced_ports", self.task.function_params, "no fence record is ever written into the task") - self.assertEqual(self.task.status, JobSchedule.STATUS_DONE) def test_suspension_leaves_no_block_behind(self): def _fail_own(peer, primary): return not (peer is self.node and primary is self.prim) - self._run_with_verify(_fail_own) - self.assertEqual(self.task.status, JobSchedule.STATUS_SUSPENDED) + with self.assertRaises(TaskDefer): + self._run_with_verify(_fail_own) self.assertEqual( self._blocks(), [], "a suspended recovery must not leave any port blocked — the " @@ -1637,12 +1621,11 @@ def test_tertiary_path_gated_before_allow(self): "the tertiary->secondary hublvol must be connected " "BEFORE the port opens — the tertiary is the acting " "leader and must be able to redirect here") - self.assertEqual(self.task.status, JobSchedule.STATUS_DONE) def test_tertiary_path_failure_suspends(self): self.tert.add_hublvol_failover_path = MagicMock(return_value=False) - self._run_with_verify(lambda *a: True) - self.assertEqual(self.task.status, JobSchedule.STATUS_SUSPENDED) + with self.assertRaises(TaskDefer): + self._run_with_verify(lambda *a: True) node_allows = [ c for c in self.calls if c[0] == "firewall_set_port" and c[1] == self.node.uuid and c[3] == "allow" @@ -1659,7 +1642,6 @@ def test_down_primary_also_gates_tertiary_path(self): self._run_with_verify(lambda *a: True) tert_paths = [c for c in self.calls if c[0] == "tert_path"] self.assertEqual(len(tert_paths), 1) - self.assertEqual(self.task.status, JobSchedule.STATUS_DONE) # --- deferred post-unblock ANA promotion (primary OFFLINE) ---------- @@ -1705,7 +1687,6 @@ def test_ana_switchback_promotes_secondary_listeners_post_unblock(self): "must never gate the recovery") self.assertLess(i_event, i_ana, "the promotion runs after the port_allowed event") - self.assertEqual(self.task.status, JobSchedule.STATUS_DONE) def test_ana_promotion_failure_is_best_effort(self): # A failing per-lvol promotion must not fail the (already @@ -1717,7 +1698,6 @@ def _boom(lvol, target, state): raise Exception("listener set-ana rpc failed") self._run_with_ana(ana_side=_boom) self.assertEqual(len(self._node_allows()), 1) - self.assertEqual(self.task.status, JobSchedule.STATUS_DONE) @unittest.skip("pending design call 2026-07: deferred ANA DEMOTE (a " @@ -1779,12 +1759,11 @@ def test_redirect_rewired_toward_acting_leader(self): self.assertIs(call.args[0], self.sec1) self.assertEqual(call.kwargs.get("role"), "tertiary") self.assertIs(call.kwargs.get("lvs_node"), self.prim2) - self.assertEqual(self.task.status, JobSchedule.STATUS_DONE) def test_failure_suspends_before_port_opens(self): self.node.connect_to_hublvol = MagicMock(return_value=False) - self._run() - self.assertEqual(self.task.status, JobSchedule.STATUS_SUSPENDED) + with self.assertRaises(TaskDefer): + self._run() node_allows = [ c for c in self.calls if c[0] == "firewall_set_port" and c[1] == self.node.uuid and c[3] == "allow" diff --git a/tests/unit/tasks/test_retry_ceiling.py b/tests/unit/tasks/test_retry_ceiling.py index 6e80bfba01..502d7cd510 100644 --- a/tests/unit/tasks/test_retry_ceiling.py +++ b/tests/unit/tasks/test_retry_ceiling.py @@ -272,6 +272,8 @@ def _get_node(node_id): "tasks_runner_migration.py", "tasks_runner_new_dev_migration.py", "tasks_runner_failed_migration.py", + "tasks_runner_node_removal.py", + "tasks_runner_port_allow.py", } # Runners that increment task.retry but are intentionally UNBOUNDED: the @@ -280,7 +282,6 @@ def _get_node(node_id): # the reason, surfaced in the skip message. INTENTIONALLY_UNBOUNDED = { "tasks_runner_lvol_migration.py": "created with max_retry=-1; retry gated on resource recovery", - "tasks_runner_node_removal.py": "created with max_retry=-1; multi-hour removal gated on failure-migration completion", } _INCREMENTS_RETRY = re.compile(r"\.retry\s*\+=\s*1") diff --git a/tests/unit/tasks/test_runner_specs.py b/tests/unit/tasks/test_runner_specs.py index 53074ebba0..7bd808d55f 100644 --- a/tests/unit/tasks/test_runner_specs.py +++ b/tests/unit/tasks/test_runner_specs.py @@ -490,3 +490,46 @@ def test_failed_migration_waits_for_the_last_task_on_the_device(failed_migration failed_migration.SPEC.on_finish(task) failed_migration.device_controller.device_set_failed_and_migrated.assert_not_called() + + +# -- every migrated runner -------------------------------------------------- + +MIGRATED_RUNNERS = [ + "fdb_backup", "jc_comp", "replication_final", "sync_lvol_del", "backup", + "cluster_expand", "node_add", "restart", "migration", "new_dev_migration", + "failed_migration", "node_removal", "port_allow", +] + + +@pytest.mark.parametrize("name", MIGRATED_RUNNERS) +def test_runner_module_defines_a_usable_spec(name): + """Import-smoke plus spec sanity: the module-level definitions still load + under the stubbed-fdb unit env, and what serve() will be handed is + actually runnable.""" + import importlib + module = importlib.import_module(f"simplyblock_core.services.tasks_runner_{name}") + + spec = module.SPEC + assert spec.function_names, f"{name}: no function names" + assert callable(spec.handler) + assert callable(spec.is_eligible) + assert spec.concurrency >= 1 + assert spec.interval > 0 + for optional in (spec.on_finish, spec.on_cycle, spec.serialize, + spec.exclusion_key, spec.backoff): + assert optional is None or callable(optional) + assert callable(module.main) + + +def test_every_task_function_has_exactly_one_runner(): + """Two runners claiming the same function name would both drive it, and + the lease only guards against a second host, not a second spec.""" + import importlib + owners = {} + for name in MIGRATED_RUNNERS: + module = importlib.import_module(f"simplyblock_core.services.tasks_runner_{name}") + for function_name in module.SPEC.function_names: + owners.setdefault(function_name, []).append(name) + + duplicates = {fn: names for fn, names in owners.items() if len(names) > 1} + assert not duplicates, f"function names claimed by more than one runner: {duplicates}" From 2523be0f9efcf4a28e15b0a928a2dab360f2b248 Mon Sep 17 00:00:00 2001 From: Max Schettler Date: Thu, 6 Aug 2026 02:12:47 +0200 Subject: [PATCH 24/25] Update handover document --- HANDOFF.md | 27 +++++++++++++-------------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/HANDOFF.md b/HANDOFF.md index 65ccf391d4..311b903435 100644 --- a/HANDOFF.md +++ b/HANDOFF.md @@ -21,25 +21,24 @@ ad4768462 Introduce generic task runner (B0) ``` -## Status: Phase A done, B0 done, 8 of 13 runners migrated +## Status: the planned scope is complete -Migrated onto the driver: `fdb_backup`, `jc_comp`, `replication_final`, -`sync_lvol_del`, `backup`, `cluster_expand`, `node_add`, `restart`. +All thirteen runners in scope are on the driver: `fdb_backup`, `jc_comp`, +`replication_final`, `sync_lvol_del`, `backup`, `cluster_expand`, `node_add`, +`restart`, `migration`, `new_dev_migration`, `failed_migration`, `node_removal`, +`port_allow`. -**Remaining, in order:** - -1. migration trio — `migration`, `new_dev_migration`, `failed_migration` - (serial; `get_active_node_mig_task` / same-node-sibling gating becomes - `is_eligible`). Note `tasks_controller.defer_if_cluster_expanding` writes - task state from inside these handlers — it becomes a `TaskDefer` (or an - `is_eligible` clause) and the helper goes away. -2. `port_allow` (large; its `is_eligible` omits the `IN_ACTIVATION` check — the - documented opt-out — and keeps the recovery logic) -3. `node_removal` (serial, already leased, unbounded) +Nothing outside a runner writes task lifecycle state any more: +`utils.handle_task_result` and `tasks_controller.defer_task_for_expansion` are +gone, and the three cancellation paths commit by CAS. **Deferred to a follow-up PR:** `lvol_migration` and `batch_migration` — under active upstream rewrite (~17 commits in the last window, several still labelled -`TEMP:`). Migrating them now guarantees repeated conflicts for no benefit. +`TEMP:`). They keep their own loops meanwhile, which is why +`tasks_cluster_status.py` is not the only file left untouched. Migrating them is +the same exercise: their pre-run guard chains become `is_eligible`, their poll +loops `TaskProgress`, and their `_suspend_task(charge_retry=...)` helper maps +directly onto TaskDefer vs TaskRetry. ## What the driver grew during the migrations From b0b97283ba747c88bae7ae9160b80c59ad046b69 Mon Sep 17 00:00:00 2001 From: Max Schettler Date: Sat, 22 Aug 2026 17:34:08 +0200 Subject: [PATCH 25/25] Report task failures through the task runner A handler that signals TaskRetry was silent: the driver logged only unexpected exceptions, so the log line every runner used to write inline before suspending was lost in the migration, for all of them at once. _fail now logs the reason. The one failure that also has to reach the cluster event log -- a sync delete failing after the async delete already succeeded, which pins the volume in_deletion with nothing in the volume list to say why -- gets a declarative hook instead of a hand-rolled one: RunnerSpec.on_failure, called with (task, reason) once per DISTINCT failure message. Recognizing a repeat is the driver's job, not the handler's, because the driver is what clears function_result before each attempt: it now keeps the previous attempt's message and compares. That restores the original dedupe exactly, and unlike the in-process map it replaces, it survives a runner restart. The alert fires only for the caller that won the transition, and a hook that raises cannot take the loop down -- both as on_finish already behaves. Claude-Session: https://claude.ai/code/session_012GeSWcbJdAX19i1yZQ6jqg --- simplyblock_core/services/task_runner_base.py | 36 ++++++++- .../services/tasks_runner_sync_lvol_del.py | 53 +++++------- tests/unit/tasks/test_task_runner_base.py | 81 +++++++++++++++++++ tests/unit/test_lifecycle_alert_events.py | 54 +++++-------- 4 files changed, 153 insertions(+), 71 deletions(-) diff --git a/simplyblock_core/services/task_runner_base.py b/simplyblock_core/services/task_runner_base.py index 99f53663cb..30c23e6049 100644 --- a/simplyblock_core/services/task_runner_base.py +++ b/simplyblock_core/services/task_runner_base.py @@ -25,7 +25,8 @@ mutual exclusion on a RUNNING sibling depend on that. - **raise** :class:`TaskRetry` (or any other, unexpected ``Exception``) — a retryable failure. Suspend, **consume a retry**, and back off before the next - attempt. + attempt. A failure whose message differs from the previous attempt's also + fires the spec's ``on_failure`` alert. - **raise** :class:`TaskAbort` — a permanent, non-retryable stop (missing param, object gone, "not needed"). Finish the task (``STATUS_DONE``) with the reason. @@ -189,6 +190,13 @@ class RunnerSpec: # otherwise leak on the terminal paths the handler never sees. Runs only # for the caller that won the terminal transition. on_finish: Optional[Callable[[JobSchedule], None]] = None + # Optional alert, called with (task, reason) when a task fails with a + # message DIFFERENT from the one its previous attempt recorded. For a + # failure an operator must see in the cluster event log rather than only in + # `sbctl task list` — a repeat of the same message is not re-reported, so a + # cause that persists for hours writes one event, not one per attempt. + # Never called for a TaskDefer: waiting on external state is not failing. + on_failure: Optional[Callable[[JobSchedule, str], None]] = None # Optional per-task, per-cycle "must this one run to completion before the # loop moves on?". Defaults to serializing exactly when the pool has a # single worker. A runner whose mode depends on live cluster state (node @@ -346,6 +354,9 @@ def _process(self, task: JobSchedule, cluster: Any) -> None: # Drop the previous attempt's result so a task that fails and later # succeeds does not finish carrying the stale failure message. Handlers # that set a success message overwrite this; the rest get "completed". + # Kept first: the handler cannot see what the last attempt reported, so + # recognizing a repeated failure is the driver's job (see _fail). + previous_result = task.function_result task.function_result = "" try: @@ -362,11 +373,11 @@ def _process(self, task: JobSchedule, cluster: Any) -> None: except TaskAbort as e: self._finish(task, str(e) or "aborted") except TaskRetry as e: - self._fail(task, str(e) or "retry") + self._fail(task, str(e) or "retry", previous_result) except Exception as e: # noqa: BLE001 - unexpected == retryable failure logger.error(f"{self.spec.name}: task {uuid} handler raised: {e}") logger.exception(e) - self._fail(task, f"unhandled error: {e}") + self._fail(task, f"unhandled error: {e}", previous_result) else: self._succeed(task) @@ -420,6 +431,17 @@ def _progress(self, task: JobSchedule, reason: str) -> None: return self._clear_backoff(task.uuid) + def _alert(self, task: JobSchedule, reason: str) -> None: + if self.spec.on_failure is None: + return + # As on_finish: a hook that cannot record its alert must not turn a + # retryable task failure into a dead runner. + try: + self.spec.on_failure(task, reason) + except Exception as e: # noqa: BLE001 - alerting failure is not fatal + logger.error(f"{self.spec.name}: task {task.uuid} on_failure failed: {e}") + logger.exception(e) + def _defer(self, task: JobSchedule, reason: str) -> None: if reason: task.function_result = reason @@ -428,7 +450,8 @@ def _defer(self, task: JobSchedule, reason: str) -> None: return self._clear_backoff(task.uuid) - def _fail(self, task: JobSchedule, reason: str) -> None: + def _fail(self, task: JobSchedule, reason: str, previous_result: str = "") -> None: + logger.error(f"{self.spec.name}: task {task.uuid} failed: {reason}") task.function_result = reason def _apply(fresh: JobSchedule) -> None: @@ -439,6 +462,11 @@ def _apply(fresh: JobSchedule) -> None: if committed is None: self._forget(task.uuid) return + # As on_finish: alert only for the caller that recorded the outcome, and + # only once per distinct message — a cause that persists for hours must + # not write one event per attempt. + if reason != previous_result: + self._alert(committed, reason) # Back off on the committed retry count, not the stale local one. with self._lock: self._next_attempt[task.uuid] = time.time() + self._backoff_delay(committed.retry) diff --git a/simplyblock_core/services/tasks_runner_sync_lvol_del.py b/simplyblock_core/services/tasks_runner_sync_lvol_del.py index 8eb13d3871..0964f03c71 100644 --- a/simplyblock_core/services/tasks_runner_sync_lvol_del.py +++ b/simplyblock_core/services/tasks_runner_sync_lvol_del.py @@ -89,38 +89,28 @@ def _run_sync_op(task): raise TaskAbort(f"unknown op {op!r}") -# Last failure message reported per delete task, so a node that stays down does -# not write one identical event per 3s poll. The driver clears ``function_result`` -# before every attempt, so the previous message cannot be read off the task. -_last_sync_del_failure: dict = {} - - -def _log_sync_delete_failure(task, node, lvol_bdev_name, msg): - """Record a failed sync delete in the cluster event log. +def alert_sync_delete_failure(task, reason): + """Report a failed sync delete in the cluster event log. This is the case an operator cannot see from the volume list alone: the async delete already SUCCEEDED, so the data is going away, but a node still - holds its replica bdev and the volume is pinned in_deletion until this - task drains. + holds its replica bdev and the volume is pinned in_deletion until this task + drains. """ - if _last_sync_del_failure.get(task.uuid) == msg: + if task.function_name != JobSchedule.FN_LVOL_SYNC_DEL: return - _last_sync_del_failure[task.uuid] = msg - try: - events_controller.log_event_cluster( - cluster_id=node.cluster_id, - domain=events_controller.DOMAIN_STORAGE, - event="SYNC_DELETE_FAILED", - db_object=task, - caused_by=events_controller.CAUSED_BY_MONITOR, - message=(f"Sync delete of {lvol_bdev_name} failed on node " - f"{node.get_id()} after a successful async delete; the " - f"volume stays in_deletion until this drains. {msg}"), - node_id=node.get_id(), - event_level="Error") - except Exception as event_error: - logger.warning(f"Could not log sync-delete failure event: {event_error}") + events_controller.log_event_cluster( + cluster_id=task.cluster_id, + domain=events_controller.DOMAIN_STORAGE, + event="SYNC_DELETE_FAILED", + db_object=task, + caused_by=events_controller.CAUSED_BY_MONITOR, + message=(f"Sync delete of {task.function_params['lvol_bdev_name']} failed on " + f"node {task.node_id} after a successful async delete; the volume " + f"stays in_deletion until this drains. {reason}"), + node_id=task.node_id, + event_level="Error") def _run_sync_del(task): @@ -144,15 +134,11 @@ def _run_sync_del(task): node_id=node.get_id()): ret, err = node.rpc_client().delete_lvol(lvol_bdev_name, sync=True) except Exception as e: - msg = f"Sync delete of {lvol_bdev_name} on {node.get_id()} failed: {e}; will retry" - _log_sync_delete_failure(task, node, lvol_bdev_name, msg) - raise TaskRetry(msg) + raise TaskRetry(f"Sync delete of {lvol_bdev_name} on {node.get_id()} failed: {e}; will retry") if not ret: if "code" not in err or err["code"] != -19: - msg = f"Failed to sync delete bdev: {lvol_bdev_name} from node: {node.get_id()}" - _log_sync_delete_failure(task, node, lvol_bdev_name, msg) - raise TaskRetry(msg) + raise TaskRetry(f"Failed to sync delete bdev: {lvol_bdev_name} from node: {node.get_id()}") logger.error(f"Sync delete completed with error: {err}") task.function_result = f"bdev {lvol_bdev_name} deleted" @@ -171,8 +157,6 @@ def release_del_sync_lock(task): if task.function_name != JobSchedule.FN_LVOL_SYNC_DEL: return - _last_sync_del_failure.pop(task.uuid, None) - primary_node = get_primary_node(task) if primary_node: primary_node.lvol_del_sync_lock_reset() @@ -183,6 +167,7 @@ def release_del_sync_lock(task): function_names=[JobSchedule.FN_LVOL_SYNC_OP, JobSchedule.FN_LVOL_SYNC_DEL], handler=process_task, on_finish=release_del_sync_lock, + on_failure=alert_sync_delete_failure, is_eligible=lambda task, cluster: cluster.status != Cluster.STATUS_IN_ACTIVATION, interval=3, ) diff --git a/tests/unit/tasks/test_task_runner_base.py b/tests/unit/tasks/test_task_runner_base.py index 13df330d0f..c04a416c2e 100644 --- a/tests/unit/tasks/test_task_runner_base.py +++ b/tests/unit/tasks/test_task_runner_base.py @@ -255,6 +255,87 @@ def test_failing_on_finish_does_not_break_the_task(monkeypatch): assert store.row().status == JobSchedule.STATUS_DONE +# -- on_failure alerts ------------------------------------------------------ + +def test_on_failure_reports_a_failed_attempt(monkeypatch): + task = _task() + store = _wire(monkeypatch, task) + on_failure = MagicMock() + + _runner(MagicMock(side_effect=trb.TaskRetry("boom")), + on_failure=on_failure)._process(task, MagicMock()) + + reported_task, reason = on_failure.call_args[0] + assert reported_task.uuid == "task-1" + assert reason == "boom" + assert store.row().status == JobSchedule.STATUS_SUSPENDED + + +def test_on_failure_is_not_repeated_for_the_same_message(monkeypatch): + """A cause that persists for hours must write ONE alert, not one per + attempt. The handler cannot tell — it is handed a cleared function_result — + so the driver compares against the message the row still carries.""" + task = _task(status=JobSchedule.STATUS_SUSPENDED) + task.function_result = "boom" + store = _wire(monkeypatch, task) + on_failure = MagicMock() + + _runner(MagicMock(side_effect=trb.TaskRetry("boom")), + on_failure=on_failure)._process(task, MagicMock()) + + on_failure.assert_not_called() + assert store.row().retry == 1 + + +def test_on_failure_reports_a_changed_message(monkeypatch): + task = _task(status=JobSchedule.STATUS_SUSPENDED) + task.function_result = "connection refused" + _wire(monkeypatch, task) + on_failure = MagicMock() + + _runner(MagicMock(side_effect=trb.TaskRetry("timeout")), + on_failure=on_failure)._process(task, MagicMock()) + + assert on_failure.call_args[0][1] == "timeout" + + +def test_on_failure_does_not_run_for_a_deferred_task(monkeypatch): + task = _task() + _wire(monkeypatch, task) + on_failure = MagicMock() + + _runner(MagicMock(side_effect=trb.TaskDefer("waiting")), + on_failure=on_failure)._process(task, MagicMock()) + + on_failure.assert_not_called() + + +def test_on_failure_does_not_run_when_another_actor_owns_the_outcome(monkeypatch): + task = _task() + store = _wire(monkeypatch, task) + on_failure = MagicMock() + + def _handler(t): + store.concurrently(canceled=True) + raise trb.TaskRetry("boom") + + _runner(_handler, on_failure=on_failure)._process(task, MagicMock()) + + on_failure.assert_not_called() + + +def test_failing_on_failure_does_not_break_the_task(monkeypatch): + task = _task() + store = _wire(monkeypatch, task) + + runner = _runner(MagicMock(side_effect=trb.TaskRetry("boom")), + on_failure=MagicMock(side_effect=RuntimeError("alert boom"))) + runner._process(task, MagicMock()) # must not raise + + assert store.row().status == JobSchedule.STATUS_SUSPENDED + assert store.row().retry == 1 + + # -- pre-run skip-gates ----------------------------------------------------- def test_ineligible_skips_without_claim_or_write(monkeypatch): diff --git a/tests/unit/test_lifecycle_alert_events.py b/tests/unit/test_lifecycle_alert_events.py index b965d07818..a55f258b18 100644 --- a/tests/unit/test_lifecycle_alert_events.py +++ b/tests/unit/test_lifecycle_alert_events.py @@ -5,7 +5,7 @@ 1. A sync delete failing after the async delete already succeeded. The data is going away, but a node still holds its replica bdev and the volume is pinned in_deletion until the deferred task drains -- invisible from the volume list - alone. The runner retries every 3 seconds, so the event fires only when the + alone. The runner declares the alert; the task runner fires it only when the failure message CHANGES, not per retry. 2. An lvs journal accumulating more than JM_COMPRESSION_BACKLOG_ALERT_RECORDS @@ -18,6 +18,7 @@ from unittest.mock import MagicMock, patch from simplyblock_core import constants +from simplyblock_core.models.job_schedule import JobSchedule from simplyblock_core.services import main_distr_event_collector as collector from simplyblock_core.services import tasks_runner_sync_lvol_del as sync_del @@ -27,55 +28,42 @@ class TestSyncDeleteFailureEvent(unittest.TestCase): + """The runner's alert hook, which the task runner calls on a failed attempt. + That it fires once per DISTINCT failure rather than once per retry, and that + a broken event log cannot take the runner down, are the driver's contracts — + see tests/unit/tasks/test_task_runner_base.py.""" - def _task(self, previous_result=""): + def _task(self, function_name=JobSchedule.FN_LVOL_SYNC_DEL, + bdev="LVS_1/LVOL_5"): task = MagicMock() task.uuid = "task-1" - # The task runner clears function_result before every attempt, so the - # "same failure again" check reads the runner's own last-message map. - sync_del._last_sync_del_failure.clear() - if previous_result: - sync_del._last_sync_del_failure[task.uuid] = previous_result + task.cluster_id = "cl-1" + task.node_id = "node-1" + task.function_name = function_name + task.function_params = {"lvol_bdev_name": bdev} return task - def _node(self): - node = MagicMock() - node.cluster_id = "cl-1" - node.get_id.return_value = "node-1" - return node - - def test_first_failure_is_logged_as_an_error(self): + def test_a_failure_is_reported_as_an_error(self): with patch.object(sync_del.events_controller, "log_event_cluster") as log: - sync_del._log_sync_delete_failure( - self._task(), self._node(), "LVS_1/LVOL_5", "connection refused") + sync_del.alert_sync_delete_failure(self._task(), "connection refused") log.assert_called_once() kwargs = log.call_args.kwargs self.assertEqual(kwargs["event_level"], "Error") self.assertEqual(kwargs["event"], "SYNC_DELETE_FAILED") self.assertEqual(kwargs["node_id"], "node-1") + self.assertEqual(kwargs["cluster_id"], "cl-1") self.assertIn("in_deletion", kwargs["message"]) self.assertIn("LVS_1/LVOL_5", kwargs["message"]) + self.assertIn("connection refused", kwargs["message"]) - def test_a_repeat_of_the_same_failure_is_not_logged_again(self): - """The runner retries every 3s; identical failures must not flood.""" - task = self._task(previous_result="boom") + def test_a_sync_op_failure_is_not_a_sync_delete_alert(self): + """Both task families share this runner, but only a delete leaves a + volume pinned in_deletion.""" with patch.object(sync_del.events_controller, "log_event_cluster") as log: - sync_del._log_sync_delete_failure(task, self._node(), "b", "boom") + sync_del.alert_sync_delete_failure( + self._task(function_name=JobSchedule.FN_LVOL_SYNC_OP), "boom") log.assert_not_called() - def test_a_different_failure_is_logged(self): - task = self._task(previous_result="connection refused") - with patch.object(sync_del.events_controller, "log_event_cluster") as log: - sync_del._log_sync_delete_failure(task, self._node(), "b", "timeout") - log.assert_called_once() - - def test_event_log_trouble_does_not_break_the_runner(self): - with patch.object(sync_del.events_controller, "log_event_cluster", - side_effect=RuntimeError("db gone")), \ - patch.object(sync_del, "logger"): - sync_del._log_sync_delete_failure( - self._task(), self._node(), "b", "boom") # must not raise - class TestCompressionBacklogEvent(unittest.TestCase):