From e418afb4fa7280c3a5594da125c9f337f7624e56 Mon Sep 17 00:00:00 2001 From: Alex Myers Date: Mon, 20 Oct 2025 14:15:39 -0500 Subject: [PATCH 01/56] reckless: report version without loading config --- tests/test_reckless.py | 12 ++++++++++++ tools/reckless | 3 +++ 2 files changed, 15 insertions(+) diff --git a/tests/test_reckless.py b/tests/test_reckless.py index d294b79a50ef..3d2d66a7758a 100644 --- a/tests/test_reckless.py +++ b/tests/test_reckless.py @@ -170,6 +170,18 @@ def test_basic_help(): assert r.search_stdout("options:") or r.search_stdout("optional arguments:") +def test_version(): + '''Version should be reported without loading config and should advance + with lightningd''' + r = reckless(["-V", "-v", "--json"]) + assert r.returncode == 0 + import json + json_out = ''.join(r.stdout) + with open('.version', 'r') as f: + version = f.readlines()[0].strip() + assert json.loads(json_out)['result'][0] == version + + def test_contextual_help(node_factory): n = get_reckless_node(node_factory) for subcmd in ['install', 'uninstall', 'search', diff --git a/tools/reckless b/tools/reckless index 0d9ea8d0393a..c4741e1f7fd1 100755 --- a/tools/reckless +++ b/tools/reckless @@ -2082,6 +2082,9 @@ if __name__ == '__main__': 'signet', 'testnet', 'testnet4'] if args.version: report_version() + if log.capture: + log.reply_json() + sys.exit(0) elif args.cmd1 is None: parser.print_help(sys.stdout) sys.exit(1) From a1d69c85463c2e7fefb903f041af0bfdb36ae3be Mon Sep 17 00:00:00 2001 From: Alex Myers Date: Mon, 20 Oct 2025 14:39:18 -0500 Subject: [PATCH 02/56] reckless: redirect help alias output when --json option is used This doesn't change the argparse behavior with --help/-h, but it does correct the output in this one case where we must manually call it. --- tools/reckless | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/tools/reckless b/tools/reckless index c4741e1f7fd1..0b216acaa266 100755 --- a/tools/reckless +++ b/tools/reckless @@ -5,6 +5,7 @@ import argparse import copy import datetime from enum import Enum +import io import json import logging import os @@ -1124,9 +1125,16 @@ INSTALLERS = [pythonuv, pythonuvlegacy, python3venv, poetryvenv, def help_alias(targets: list): if len(targets) == 0: - parser.print_help(sys.stdout) + if log.capture: + help_output = io.StringIO() + parser.print_help(help_output) + log.add_result(help_output.getvalue()) + else: + parser.print_help(sys.stdout) else: log.info('try "reckless {} -h"'.format(' '.join(targets))) + if log.capture: + log.reply_json() sys.exit(1) @@ -2124,7 +2132,9 @@ if __name__ == '__main__': if 'targets' in args: # and len(args.targets) > 0: if args.func.__name__ == 'help_alias': - args.func(args.targets) + log.add_result(args.func(args.targets)) + if log.capture: + log.reply_json() sys.exit(0) # Catch a missing argument so that we can overload functions. if len(args.targets) == 0: From e004d95aad17867994ce03bfd0395cfc0623534c Mon Sep 17 00:00:00 2001 From: Alex Myers Date: Tue, 21 Oct 2025 14:13:52 -0500 Subject: [PATCH 03/56] reckless: handle .git in source locations --- tools/reckless | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/tools/reckless b/tools/reckless index 0b216acaa266..464c5370e757 100755 --- a/tools/reckless +++ b/tools/reckless @@ -277,14 +277,14 @@ class InstInfo: pass # If unable to search deeper, resort to matching directory name elif recursion < 1: - if sub.name.lower() == self.name.lower(): + if sub.name.lower().removesuffix('.git') == self.name.lower(): # Partial success (can't check for entrypoint) self.name = sub.name return sub return None sub.populate() - if sub.name.lower() == self.name.lower(): + if sub.name.lower().removesuffix('.git') == self.name.lower(): # Directory matches the name we're trying to install, so check # for entrypoint and dependencies. for inst in INSTALLERS: @@ -302,7 +302,7 @@ class InstInfo: self.entry = found_entry.name self.deps = found_dep.name return sub - log.debug(f"missing dependency for {self}") + log.debug(f"{inst.name} installer: missing dependency for {self}") found_entry = None for file in sub.contents: if isinstance(file, SourceDir): @@ -404,7 +404,7 @@ class Source(Enum): trailing = Path(source.lower().partition('github.com/')[2]).parts if len(trailing) < 2: return None, None - return trailing[0], trailing[1] + return trailing[0], trailing[1].removesuffix('.git') class SourceDir(): @@ -451,7 +451,7 @@ class SourceDir(): for c in self.contents: if ftype and not isinstance(c, ftype): continue - if c.name.lower() == name.lower(): + if c.name.lower().removesuffix('.git') == name.lower(): return c return None @@ -627,7 +627,7 @@ def populate_github_repo(url: str) -> list: while '' in repo: repo.remove('') repo_name = None - parsed_url = urlparse(url) + parsed_url = urlparse(url.removesuffix('.git')) if 'github.com' not in parsed_url.netloc: return None if len(parsed_url.path.split('/')) < 2: @@ -1212,7 +1212,7 @@ def _git_update(github_source: InstInfo, local_copy: PosixPath): if git.returncode != 0: return False default_branch = git.stdout.splitlines()[0] - if default_branch != 'origin/master': + if default_branch not in ['origin/master', 'origin/main']: log.debug(f'UNUSUAL: fetched default branch {default_branch} for ' f'{github_source.source_loc}') @@ -1589,7 +1589,7 @@ def search(plugin_name: str) -> Union[InstInfo, None]: for src in RECKLESS_SOURCES: # Search repos named after the plugin before collections if Source.get_type(src) == Source.GITHUB_REPO: - if src.split('/')[-1].lower() == plugin_name.lower(): + if src.split('/')[-1].lower().removesuffix('.git') == plugin_name.lower(): ordered_sources.remove(src) ordered_sources.insert(0, src) # Check locally before reaching out to remote repositories From a7a0633fead4b7a0db6bf3289c1b21c00e86ba0a Mon Sep 17 00:00:00 2001 From: Alex Myers Date: Thu, 23 Oct 2025 12:24:48 -0500 Subject: [PATCH 04/56] reckless: cleanup failed installation attempts This is needed when installation is managed by an application that may not have access to the filesystem to clean up manually. --- tests/test_reckless.py | 12 ++++++++++++ tools/reckless | 29 +++++++++++++++++++++++++---- 2 files changed, 37 insertions(+), 4 deletions(-) diff --git a/tests/test_reckless.py b/tests/test_reckless.py index 3d2d66a7758a..6dd479d45bbf 100644 --- a/tests/test_reckless.py +++ b/tests/test_reckless.py @@ -276,6 +276,18 @@ def test_install(node_factory): assert os.path.exists(plugin_path) +def test_install_cleanup(node_factory): + """test failed installation and post install cleanup""" + n = get_reckless_node(node_factory) + n.start() + r = reckless([f"--network={NETWORK}", "-v", "install", "testplugfail"], dir=n.lightning_dir) + assert r.returncode == 0 + assert r.search_stdout('testplugfail failed to start') + r.check_stderr() + plugin_path = Path(n.lightning_dir) / 'reckless/testplugfail' + assert not os.path.exists(plugin_path) + + @unittest.skipIf(VALGRIND, "virtual environment triggers memleak detection") def test_poetry_install(node_factory): """test search, git clone, and installation to folder.""" diff --git a/tools/reckless b/tools/reckless index 464c5370e757..85fa4cdd702e 100755 --- a/tools/reckless +++ b/tools/reckless @@ -1484,10 +1484,22 @@ def _enable_installed(installed: InstInfo, plugin_name: str) -> Union[str, None] if enable(installed.name): return f"{installed.source_loc}" - log.error(('dynamic activation failed: ' - f'{installed.name} not found in reckless directory')) + log.error('dynamic activation failed') return None + +def cleanup_plugin_installation(plugin_name): + """Remove traces of an installation attempt.""" + inst_path = Path(RECKLESS_CONFIG.reckless_dir) / plugin_name + if not inst_path.exists(): + log.warning(f'asked to clean up {inst_path}, but nothing is present.') + return + + log.info(f'Cleaning up partial installation of {plugin_name} at {inst_path}') + shutil.rmtree(inst_path) + return + + def install(plugin_name: str) -> Union[str, None]: """Downloads plugin from source repos, installs and activates plugin. Returns the location of the installed plugin or "None" in the case of @@ -1504,7 +1516,7 @@ def install(plugin_name: str) -> Union[str, None]: direct_location, name = location_from_name(name) src = None if direct_location: - logging.debug(f"install of {name} requested from {direct_location}") + log.debug(f"install of {name} requested from {direct_location}") src = InstInfo(name, direct_location, name) # Treating a local git repo as a directory allows testing # uncommitted changes. @@ -1529,8 +1541,17 @@ def install(plugin_name: str) -> Union[str, None]: except FileExistsError as err: log.error(f'File exists: {err.filename}') return None - return _enable_installed(installed, plugin_name) + except InstallationFailure as err: + cleanup_plugin_installation(plugin_name) + if log.capture: + log.warning(err) + return None + raise err + result = _enable_installed(installed, plugin_name) + if not result: + cleanup_plugin_installation(plugin_name) + return result def uninstall(plugin_name: str) -> str: From fbe6885548dce34aa23b79ff1a411c06a2ee6afc Mon Sep 17 00:00:00 2001 From: Alex Myers Date: Thu, 23 Oct 2025 16:40:14 -0500 Subject: [PATCH 05/56] reckless: add listinstalled command to list reckless managed plugins 'Reckless listinstalled' will now list all plugins installed and managed by reckless. Changelog-Added: reckless: `listinstalled` command lists plugins installed by reckless. --- tools/reckless | 92 +++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 91 insertions(+), 1 deletion(-) diff --git a/tools/reckless b/tools/reckless index 85fa4cdd702e..0c8b2d9b4ef6 100755 --- a/tools/reckless +++ b/tools/reckless @@ -1958,6 +1958,93 @@ def update_plugins(plugin_name: str): return update_results +MD_FORMAT = {'installation date': "None", + 'installation time': "None", + 'original source': "None", + 'requested commit': "None", + 'installed commit': "None", + } + + +def extract_metadata(plugin_name: str) -> dict: + metadata_file = Path(RECKLESS_CONFIG.reckless_dir) / plugin_name / '.metadata' + if not metadata_file.exists(): + return None + + with open(metadata_file, 'r') as md: + lines = md.readlines() + metadata = MD_FORMAT.copy() + current_key = None + + for line in lines: + if line.strip() in metadata: + current_key = line.strip() + continue + + if current_key: + metadata.update({current_key: line.strip()}) + current_key = None + + return metadata + + +def listinstalled(): + """list all plugins currently managed by reckless""" + dir_contents = os.listdir(RECKLESS_CONFIG.reckless_dir) + plugins = {} + for plugin in dir_contents: + if (Path(RECKLESS_CONFIG.reckless_dir) / plugin).is_dir(): + # skip hidden dirs such as reckless' .remote_sources + if plugin[0] == '.': + continue + plugins.update({plugin: None}) + + # Format output in a simple table + name_len = 0 + inst_len = 0 + for plugin in plugins.keys(): + md = extract_metadata(plugin) + name_len = max(name_len, len(plugin) + 1) + if md: + inst_len = max(inst_len, len(md['installed commit']) + 1) + else: + inst_len = max(inst_len, 5) + for plugin in plugins.keys(): + md = extract_metadata(plugin) + # Older installed plugins may be missing a .metadata file + if not md: + md = MD_FORMAT.copy() + try: + installed = InferInstall(plugin) + except: + log.debug(f'no plugin detected in directory {plugin}') + continue + + status = "unmanaged" + for line in RECKLESS_CONFIG.content: + if installed.entry in line.strip() : + if line.strip()[:7] == 'plugin=': + status = "enabled" + elif line.strip()[:15] == 'disable-plugin=': + status = "disabled" + else: + print(f'cant handle {line}') + log.info(f"{plugin:<{name_len}} {md['installed commit']:<{inst_len}} " + f"{md['installation date']:<11} {status}") + # This doesn't originate from the metadata, but we want to provide enabled status for json output + md['enabled'] = status == "enabled" + md['entrypoint'] = installed.entry + # Format for json output + for key in md: + if md[key] == 'None': + md[key] = None + if key == 'installation time' and md[key]: + md[key] = int(md[key]) + plugins[plugin] = {k.replace(' ', '_'): v for k, v in md.items()} + + return plugins + + def report_version() -> str: """return reckless version""" log.info(__VERSION__) @@ -2057,6 +2144,9 @@ if __name__ == '__main__': update.add_argument('targets', type=str, nargs='*') update.set_defaults(func=update_plugins) + list_cmd = cmd1.add_parser('listinstalled', help='list reckless-installed plugins') + list_cmd.set_defaults(func=listinstalled) + help_cmd = cmd1.add_parser('help', help='for contextual help, use ' '"reckless -h"') help_cmd.add_argument('targets', type=str, nargs='*') @@ -2067,7 +2157,7 @@ if __name__ == '__main__': all_parsers = [parser, install_cmd, uninstall_cmd, search_cmd, enable_cmd, disable_cmd, list_parse, source_add, source_rem, help_cmd, - update] + update, list_cmd] for p in all_parsers: # This default depends on the .lightning directory p.add_argument('-d', '--reckless-dir', action=StoreIdempotent, From 4316c8fc5f2ea676df58915da05e7db4b5351f1f Mon Sep 17 00:00:00 2001 From: Alex Myers Date: Tue, 28 Oct 2025 12:26:25 -0500 Subject: [PATCH 06/56] reckless: helper function for accessing local clone of remote repo --- tools/reckless | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/tools/reckless b/tools/reckless index 0c8b2d9b4ef6..ee863eac6b2b 100755 --- a/tools/reckless +++ b/tools/reckless @@ -1138,6 +1138,16 @@ def help_alias(targets: list): sys.exit(1) +def _get_local_clone(source: str) -> Union[Path, None]: + """Returns the path of a local repository clone of a github source. If one + already exists, prefer searching that to accessing the github API.""" + user, repo = Source.get_github_user_repo(source) + local_clone_location = RECKLESS_DIR / '.remote_sources' / user / repo + if local_clone_location.exists(): + return local_clone_location + return None + + def _source_search(name: str, src: str) -> Union[InstInfo, None]: """Identify source type, retrieve contents, and populate InstInfo if the relevant contents are found.""" @@ -1147,18 +1157,11 @@ def _source_search(name: str, src: str) -> Union[InstInfo, None]: # If a local clone of a github source already exists, prefer searching # that instead of accessing the github API. if source.srctype == Source.GITHUB_REPO: - # Do we have a local copy already? Use that. - user, repo = Source.get_github_user_repo(src) - assert user - assert repo - local_clone_location = RECKLESS_DIR / '.remote_sources' / user / repo - if local_clone_location.exists(): - # Make sure it's the correct remote source and fetch any updates. - if _git_update(source, local_clone_location): - log.debug(f"Using local clone of {src}: " - f"{local_clone_location}") - source.source_loc = str(local_clone_location) - source.srctype = Source.GIT_LOCAL_CLONE + local_clone = _get_local_clone(source) + if local_clone and _git_update(source, local_clone): + log.debug(f"Using local clone of {src}: {local_clone}") + source.source_loc = str(local_clone) + source.srctype = Source.GIT_LOCAL_CLONE if source.get_inst_details(): return source From 05473cfafb1e3cbcca89477cb2f3af9d6153708a Mon Sep 17 00:00:00 2001 From: Alex Myers Date: Thu, 30 Oct 2025 16:19:40 -0500 Subject: [PATCH 07/56] reckless: only fetch cloned repositorys once Keep state on the clone and subdirectories so that subsequent access doesn't try fetching again. --- tools/reckless | 398 ++++++++++++++++++++++++++++--------------------- 1 file changed, 227 insertions(+), 171 deletions(-) diff --git a/tools/reckless b/tools/reckless index ee863eac6b2b..fab3bc7be54f 100755 --- a/tools/reckless +++ b/tools/reckless @@ -206,146 +206,6 @@ class Installer: return copy.deepcopy(self) -class InstInfo: - def __init__(self, name: str, location: str, git_url: str): - self.name = name - self.source_loc = str(location) # Used for 'git clone' - self.git_url: str = git_url # API access for github repos - self.srctype: Source = Source.get_type(location) - self.entry: SourceFile = None # relative to source_loc or subdir - self.deps: str = None - self.subdir: str = None - self.commit: str = None - - def __repr__(self): - return (f'InstInfo({self.name}, {self.source_loc}, {self.git_url}, ' - f'{self.entry}, {self.deps}, {self.subdir})') - - def get_repo_commit(self) -> Union[str, None]: - """The latest commit from a remote repo or the HEAD of a local repo.""" - if self.srctype in [Source.LOCAL_REPO, Source.GIT_LOCAL_CLONE]: - git = run(['git', 'rev-parse', 'HEAD'], cwd=str(self.source_loc), - stdout=PIPE, stderr=PIPE, text=True, check=False, timeout=10) - if git.returncode != 0: - return None - return git.stdout.splitlines()[0] - - if self.srctype == Source.GITHUB_REPO: - parsed_url = urlparse(self.source_loc) - if 'github.com' not in parsed_url.netloc: - return None - if len(parsed_url.path.split('/')) < 2: - return None - start = 1 - # Maybe we were passed an api.github.com/repo/ url - if 'api' in parsed_url.netloc: - start += 1 - repo_user = parsed_url.path.split('/')[start] - repo_name = parsed_url.path.split('/')[start + 1] - api_url = f'{API_GITHUB_COM}/repos/{repo_user}/{repo_name}/commits?ref=HEAD' - r = urlopen(api_url, timeout=5) - if r.status != 200: - return None - try: - return json.loads(r.read().decode())['0']['sha'] - except: - return None - - def get_inst_details(self) -> bool: - """Search the source_loc for plugin install details. - This may be necessary if a contents api is unavailable. - Extracts entrypoint and dependencies if searchable, otherwise - matches a directory to the plugin name and stops.""" - if self.srctype == Source.DIRECTORY: - assert Path(self.source_loc).exists() - assert os.path.isdir(self.source_loc) - target = SourceDir(self.source_loc, srctype=self.srctype) - # Set recursion for how many directories deep we should search - depth = 0 - if self.srctype in [Source.DIRECTORY, Source.LOCAL_REPO, - Source.GIT_LOCAL_CLONE]: - depth = 5 - elif self.srctype == Source.GITHUB_REPO: - depth = 1 - - def search_dir(self, sub: SourceDir, subdir: bool, - recursion: int) -> Union[SourceDir, None]: - assert isinstance(recursion, int) - # carveout for archived plugins in lightningd/plugins. Other repos - # are only searched by API at the top level. - if recursion == 0 and 'archive' in sub.name.lower(): - pass - # If unable to search deeper, resort to matching directory name - elif recursion < 1: - if sub.name.lower().removesuffix('.git') == self.name.lower(): - # Partial success (can't check for entrypoint) - self.name = sub.name - return sub - return None - sub.populate() - - if sub.name.lower().removesuffix('.git') == self.name.lower(): - # Directory matches the name we're trying to install, so check - # for entrypoint and dependencies. - for inst in INSTALLERS: - for g in inst.get_entrypoints(self.name): - found_entry = sub.find(g, ftype=SourceFile) - if found_entry: - break - # FIXME: handle a list of dependencies - found_dep = sub.find(inst.dependency_file, - ftype=SourceFile) - if found_entry: - # Success! - if found_dep: - self.name = sub.name - self.entry = found_entry.name - self.deps = found_dep.name - return sub - log.debug(f"{inst.name} installer: missing dependency for {self}") - found_entry = None - for file in sub.contents: - if isinstance(file, SourceDir): - assert file.relative - success = search_dir(self, file, True, recursion - 1) - if success: - return success - return None - - try: - result = search_dir(self, target, False, depth) - # Using the rest API of github.com may result in a - # "Error 403: rate limit exceeded" or other access issues. - # Fall back to cloning and searching the local copy instead. - except HTTPError: - result = None - if self.srctype == Source.GITHUB_REPO: - # clone source to reckless dir - target = copy_remote_git_source(self) - if not target: - log.warning(f"could not clone github source {self}") - return False - log.debug(f"falling back to cloning remote repo {self}") - # Update to reflect use of a local clone - self.source_loc = str(target.location) - self.srctype = target.srctype - result = search_dir(self, target, False, 5) - - if not result: - return False - - if result: - if result != target: - if result.relative: - self.subdir = result.relative - else: - # populate() should always assign a relative path - # if not in the top-level source directory - assert self.subdir == result.name - return True - return False - - def create_dir(directory: PosixPath) -> bool: try: Path(directory).mkdir(parents=False, exist_ok=True) @@ -407,10 +267,41 @@ class Source(Enum): return trailing[0], trailing[1].removesuffix('.git') +class SubmoduleSource: + """Allows us to only fetch submodules once.""" + def __init__(self, location: str): + self.location = str(location) + self.local_clone = None + self.clone_fetched = False + + def __repr__(self): + return f'' + + +class LoadedSource: + """Allows loading all sources only once per call of reckless. Initialized + with a single line of the reckless .sources file. Keeping state also allows + minimizing API calls and refetching repositories.""" + def __init__(self, source: str): + self.original_source = source + self.type = Source.get_type(source) + self.content = SourceDir(source, self.type) + self.local_clone = None + self.local_clone_fetched = False + if self.type == Source.GITHUB_REPO: + local = _get_local_clone(source) + if local: + self.local_clone = SourceDir(local, Source.GIT_LOCAL_CLONE) + self.local_clone.parent_source = self + + def __repr__(self): + return f'' + + class SourceDir(): """Structure to search source contents.""" def __init__(self, location: str, srctype: Source = None, name: str = None, - relative: str = None): + relative: str = None, parent_source: LoadedSource = None): self.location = str(location) if name: self.name = name @@ -420,6 +311,7 @@ class SourceDir(): self.srctype = srctype self.prepopulated = False self.relative = relative # location relative to source + self.parent_source = parent_source def populate(self): """populates contents of the directory at least one level""" @@ -430,7 +322,7 @@ class SourceDir(): if self.srctype == Source.DIRECTORY: self.contents = populate_local_dir(self.location) elif self.srctype in [Source.LOCAL_REPO, Source.GIT_LOCAL_CLONE]: - self.contents = populate_local_repo(self.location) + self.contents = populate_local_repo(self.location, parent_source=self.parent_source) elif self.srctype == Source.GITHUB_REPO: self.contents = populate_github_repo(self.location) else: @@ -484,6 +376,153 @@ class SourceFile(): return False +class InstInfo: + def __init__(self, name: str, location: str, git_url: str, source_dir: SourceDir=None): + self.name = name + self.source_loc = str(location) # Used for 'git clone' + self.source_dir = source_dir # Use this insead of source_loc to only fetch once. + self.git_url: str = git_url # API access for github repos + self.srctype: Source = Source.get_type(location) + self.entry: SourceFile = None # relative to source_loc or subdir + self.deps: str = None + self.subdir: str = None + self.commit: str = None + + def __repr__(self): + return (f'InstInfo({self.name}, {self.source_loc}, {self.git_url}, ' + f'{self.entry}, {self.deps}, {self.subdir})') + + def get_repo_commit(self) -> Union[str, None]: + """The latest commit from a remote repo or the HEAD of a local repo.""" + if self.srctype in [Source.LOCAL_REPO, Source.GIT_LOCAL_CLONE]: + git = run(['git', 'rev-parse', 'HEAD'], cwd=str(self.source_loc), + stdout=PIPE, stderr=PIPE, text=True, check=False, timeout=10) + if git.returncode != 0: + return None + return git.stdout.splitlines()[0] + + if self.srctype == Source.GITHUB_REPO: + parsed_url = urlparse(self.source_loc) + if 'github.com' not in parsed_url.netloc: + return None + if len(parsed_url.path.split('/')) < 2: + return None + start = 1 + # Maybe we were passed an api.github.com/repo/ url + if 'api' in parsed_url.netloc: + start += 1 + repo_user = parsed_url.path.split('/')[start] + repo_name = parsed_url.path.split('/')[start + 1] + api_url = f'{API_GITHUB_COM}/repos/{repo_user}/{repo_name}/commits?ref=HEAD' + r = urlopen(api_url, timeout=5) + if r.status != 200: + return None + try: + return json.loads(r.read().decode())['0']['sha'] + except: + return None + + def get_inst_details(self, permissive: bool=False) -> bool: + """Search the source_loc for plugin install details. + This may be necessary if a contents api is unavailable. + Extracts entrypoint and dependencies if searchable, otherwise + matches a directory to the plugin name and stops. + permissive: allows search to sometimes match directory name only for + faster searching of remote repositorys.""" + if self.srctype == Source.DIRECTORY: + assert Path(self.source_loc).exists() + assert os.path.isdir(self.source_loc) + target = self.source_dir + if not target: + target = SourceDir(self.source_loc, srctype=self.srctype) + # Set recursion for how many directories deep we should search + depth = 0 + if self.srctype in [Source.DIRECTORY, Source.LOCAL_REPO, + Source.GIT_LOCAL_CLONE]: + depth = 5 + elif self.srctype == Source.GITHUB_REPO: + depth = 1 + + def search_dir(self, sub: SourceDir, subdir: bool, + recursion: int) -> Union[SourceDir, None]: + assert isinstance(recursion, int) + # carveout for archived plugins in lightningd/plugins. Other repos + # are only searched by API at the top level. + if recursion == 0 and 'archive' in sub.name.lower(): + pass + # If unable to search deeper, resort to matching directory name + elif recursion < 1 and permissive: + if sub.name.lower().removesuffix('.git') == self.name.lower(): + # Partial success (can't check for entrypoint) + self.name = sub.name + return sub + return None + if not sub.contents and not sub.prepopulated: + sub.populate() + + if sub.name.lower().removesuffix('.git') == self.name.lower(): + # Directory matches the name we're trying to install, so check + # for entrypoint and dependencies. + for inst in INSTALLERS: + for g in inst.get_entrypoints(self.name): + found_entry = sub.find(g, ftype=SourceFile) + if found_entry: + break + # FIXME: handle a list of dependencies + found_dep = sub.find(inst.dependency_file, + ftype=SourceFile) + if found_entry: + # Success! + if found_dep: + self.name = sub.name + self.entry = found_entry.name + self.deps = found_dep.name + return sub + if permissive is True: + log.debug(f"{inst.name} installer: missing dependency for {self}") + found_entry = None + for file in sub.contents: + if isinstance(file, SourceDir): + assert file.relative + success = search_dir(self, file, True, recursion - 1) + if success: + return success + return None + + try: + result = search_dir(self, target, False, depth) + # Using the rest API of github.com may result in a + # "Error 403: rate limit exceeded" or other access issues. + # Fall back to cloning and searching the local copy instead. + except HTTPError: + result = None + if self.srctype == Source.GITHUB_REPO: + # clone source to reckless dir + target = copy_remote_git_source(self) + if not target: + log.warning(f"could not clone github source {self}") + return False + log.debug(f"falling back to cloning remote repo {self}") + # Update to reflect use of a local clone + self.source_loc = str(target.location) + self.srctype = target.srctype + result = search_dir(self, target, False, 5) + + if not result: + return False + + if result: + if result != target: + if result.relative: + self.subdir = result.relative + else: + # populate() should always assign a relative path + # if not in the top-level source directory + assert self.subdir == result.name + return True + return False + + def populate_local_dir(path: str) -> list: assert Path(os.path.realpath(path)).exists() contents = [] @@ -497,7 +536,7 @@ def populate_local_dir(path: str) -> list: return contents -def populate_local_repo(path: str, parent=None) -> list: +def populate_local_repo(path: str, parent=None, parent_source=None) -> list: assert Path(os.path.realpath(path)).exists() if parent is None: basedir = SourceDir('base') @@ -571,10 +610,13 @@ def populate_local_repo(path: str, parent=None) -> list: relative_path = str(Path(basedir.relative) / filepath) assert relative_path submodule_dir = SourceDir(filepath, srctype=Source.LOCAL_REPO, - relative=relative_path) - populate_local_repo(Path(path) / filepath, parent=submodule_dir) + relative=relative_path, + parent_source=parent_source) + populate_local_repo(Path(path) / filepath, parent=submodule_dir, + parent_source=parent_source) submodule_dir.prepopulated = True basedir.contents.append(submodule_dir) + # parent_source.submodules.append(submodule_dir) else: populate_source_path(basedir, Path(filepath)) return basedir.contents @@ -681,7 +723,8 @@ def copy_remote_git_source(github_source: InstInfo): local_path = local_path / repo if local_path.exists(): # Fetch the latest - assert _git_update(github_source, local_path) + # FIXME: pass LoadedSource and check fetch status + assert _git_update(github_source.source_loc, local_path) else: _git_clone(github_source, local_path) return SourceDir(local_path, srctype=Source.GIT_LOCAL_CLONE) @@ -1148,22 +1191,26 @@ def _get_local_clone(source: str) -> Union[Path, None]: return None -def _source_search(name: str, src: str) -> Union[InstInfo, None]: +def _source_search(name: str, src: LoadedSource) -> Union[InstInfo, None]: """Identify source type, retrieve contents, and populate InstInfo if the relevant contents are found.""" - root_dir = SourceDir(src) + root_dir = src.content source = InstInfo(name, root_dir.location, None) # If a local clone of a github source already exists, prefer searching # that instead of accessing the github API. - if source.srctype == Source.GITHUB_REPO: - local_clone = _get_local_clone(source) - if local_clone and _git_update(source, local_clone): - log.debug(f"Using local clone of {src}: {local_clone}") - source.source_loc = str(local_clone) + if src.type == Source.GITHUB_REPO: + if src.local_clone: + if not src.local_clone_fetched: + # FIXME: Pass the LoadedSource here? + if _git_update(src.original_source, src.local_clone.location): + src.local_clone_fetched = True + log.debug(f'fetching local clone of {src.original_source}') + log.debug(f"Using local clone of {src}: {src.local_clone.location}") + source.source_loc = str(src.local_clone.location) source.srctype = Source.GIT_LOCAL_CLONE - if source.get_inst_details(): + if source.get_inst_details(permissive=True): return source return None @@ -1190,9 +1237,9 @@ def _git_clone(src: InstInfo, dest: Union[PosixPath, str]) -> bool: return True -def _git_update(github_source: InstInfo, local_copy: PosixPath): +def _git_update(github_source: str, local_copy: PosixPath): # Ensure this is the correct source - git = run(['git', 'remote', 'set-url', 'origin', github_source.source_loc], + git = run(['git', 'remote', 'set-url', 'origin', github_source], cwd=str(local_copy), stdout=PIPE, stderr=PIPE, text=True, check=False, timeout=60) assert git.returncode == 0 @@ -1217,7 +1264,7 @@ def _git_update(github_source: InstInfo, local_copy: PosixPath): default_branch = git.stdout.splitlines()[0] if default_branch not in ['origin/master', 'origin/main']: log.debug(f'UNUSUAL: fetched default branch {default_branch} for ' - f'{github_source.source_loc}') + f'{github_source}') # Checkout default branch git = run(['git', 'checkout', default_branch], @@ -1337,7 +1384,11 @@ def _install_plugin(src: InstInfo) -> Union[InstInfo, None]: # FIXME: Validate path was cloned successfully. # Depending on how we accessed the original source, there may be install # details missing. Searching the cloned repo makes sure we have it. - cloned_src = _source_search(src.name, str(clone_path)) + clone = LoadedSource(plugin_path) + clone.content.populate() + # Make sure we don't try to fetch again! + assert clone.type in [Source.DIRECTORY, Source.LOCAL_REPO] + cloned_src = _source_search(src.name, clone) log.debug(f'cloned_src: {cloned_src}') if not cloned_src: log.warning('failed to find plugin after cloning repo.') @@ -1525,7 +1576,7 @@ def install(plugin_name: str) -> Union[str, None]: # uncommitted changes. if src and src.srctype == Source.LOCAL_REPO: src.srctype = Source.DIRECTORY - if not src.get_inst_details(): + if not src.get_inst_details(permissive=True): src = None if not direct_location or not src: log.debug(f"Searching for {name}") @@ -1596,7 +1647,7 @@ def _get_all_plugins_from_source(src: str) -> list: return plugins plugins.append((root.name, src)) - + for item in root.contents: if isinstance(item, SourceDir): # Skip archive directories @@ -1612,20 +1663,20 @@ def search(plugin_name: str) -> Union[InstInfo, None]: for src in RECKLESS_SOURCES: # Search repos named after the plugin before collections - if Source.get_type(src) == Source.GITHUB_REPO: - if src.split('/')[-1].lower().removesuffix('.git') == plugin_name.lower(): + if src.type == Source.GITHUB_REPO: + if src.original_source.split('/')[-1].lower().removesuffix('.git') == plugin_name.lower(): ordered_sources.remove(src) ordered_sources.insert(0, src) # Check locally before reaching out to remote repositories for src in RECKLESS_SOURCES: - if Source.get_type(src) in [Source.DIRECTORY, Source.LOCAL_REPO]: + if src.type in [Source.DIRECTORY, Source.LOCAL_REPO]: ordered_sources.remove(src) ordered_sources.insert(0, src) # First, collect all partial matches to display to user partial_matches = [] for source in ordered_sources: - for plugin_name_found, src_url in _get_all_plugins_from_source(source): + for plugin_name_found, src_url in _get_all_plugins_from_source(source.original_source): if plugin_name.lower() in plugin_name_found.lower(): partial_matches.append((plugin_name_found, src_url)) @@ -1638,12 +1689,11 @@ def search(plugin_name: str) -> Union[InstInfo, None]: # Now try exact match for installation purposes exact_match = None for source in ordered_sources: - srctype = Source.get_type(source) - if srctype == Source.UNKNOWN: - log.debug(f'cannot search {srctype} {source}') + if source.type == Source.UNKNOWN: + log.debug(f'cannot search {source.type} {source.original_source}') continue - if srctype in [Source.DIRECTORY, Source.LOCAL_REPO, - Source.GITHUB_REPO, Source.OTHER_URL]: + if source.type in [Source.DIRECTORY, Source.LOCAL_REPO, + Source.GITHUB_REPO, Source.OTHER_URL]: found = _source_search(plugin_name, source) if found: log.debug(f"{found}, {found.srctype}") @@ -1835,8 +1885,14 @@ def load_sources() -> list: log.debug('Warning: Reckless requires write access') Config(path=str(sources_file), default_text='https://github.com/lightningd/plugins') - return ['https://github.com/lightningd/plugins'] - return sources_from_file() + sources = ['https://github.com/lightningd/plugins'] + else: + sources = sources_from_file() + + all_sources = [] + for src in sources: + all_sources.append(LoadedSource(src)) + return all_sources def add_source(src: str): From 37cbcbea60492063ab743b4a3af21839fdf70522 Mon Sep 17 00:00:00 2001 From: Alex Myers Date: Thu, 13 Nov 2025 14:16:00 -0600 Subject: [PATCH 08/56] recklessrpc: allow single term commands Allows listconfig, listinstalled, and listavailable to be called via rpc. Also allow processing non-array result in listconfig output. --- contrib/msggen/msggen/schema.json | 19 ++++++++---- doc/reckless.7.md | 5 ++-- doc/schemas/reckless.json | 19 ++++++++---- plugins/recklessrpc.c | 49 +++++++++++++++++++++++-------- 4 files changed, 66 insertions(+), 26 deletions(-) diff --git a/contrib/msggen/msggen/schema.json b/contrib/msggen/msggen/schema.json index dc2744813c27..093a98666d12 100644 --- a/contrib/msggen/msggen/schema.json +++ b/contrib/msggen/msggen/schema.json @@ -30549,12 +30549,19 @@ "additionalProperties": false, "properties": { "result": { - "type": "array", - "items": { - "type": "string" - }, - "description": [ - "Output of the requested reckless command." + "oneOf": [ + { + "type": "array", + "description": [ + "Output of the requested reckless command." + ] + }, + { + "type": "object", + "description": [ + "Output of the requested reckless command." + ] + } ] }, "log": { diff --git a/doc/reckless.7.md b/doc/reckless.7.md index 49918023d56b..acf5f4cd40d9 100644 --- a/doc/reckless.7.md +++ b/doc/reckless.7.md @@ -28,8 +28,9 @@ RETURN VALUE On success, an object is returned, containing: -- **result** (array of strings): Output of the requested reckless command.: - - (string, optional) +- **result** (one of): + - (array): Output of the requested reckless command. + - (object): Output of the requested reckless command.: - **log** (array of strings): Verbose log entries of the requested reckless command.: - (string, optional) diff --git a/doc/schemas/reckless.json b/doc/schemas/reckless.json index 294fdf8f9690..372eb772dd3e 100644 --- a/doc/schemas/reckless.json +++ b/doc/schemas/reckless.json @@ -67,12 +67,19 @@ "additionalProperties": false, "properties": { "result": { - "type": "array", - "items": { - "type": "string" - }, - "description": [ - "Output of the requested reckless command." + "oneOf": [ + { + "type": "array", + "description": [ + "Output of the requested reckless command." + ] + }, + { + "type": "object", + "description": [ + "Output of the requested reckless command." + ] + } ] }, "log": { diff --git a/plugins/recklessrpc.c b/plugins/recklessrpc.c index 5d3ebb5b2525..4815deb6d6d9 100644 --- a/plugins/recklessrpc.c +++ b/plugins/recklessrpc.c @@ -68,7 +68,7 @@ static struct command_result *reckless_result(struct io_conn *conn, reckless->process_failed); return command_finished(reckless->cmd, response); } - const jsmntok_t *results, *result, *logs, *log; + const jsmntok_t *results, *result, *logs, *log, *conf; size_t i; jsmn_parser parser; jsmntok_t *toks; @@ -97,15 +97,26 @@ static struct command_result *reckless_result(struct io_conn *conn, } response = jsonrpc_stream_success(reckless->cmd); - json_array_start(response, "result"); results = json_get_member(reckless->stdoutbuf, toks, "result"); - json_for_each_arr(i, result, results) { - json_add_string(response, - NULL, - json_strdup(reckless, reckless->stdoutbuf, - result)); + conf = json_get_member(reckless->stdoutbuf, results, "requested_lightning_conf"); + if (conf) { + plugin_log(plugin, LOG_DBG, "dealing with listconfigs output"); + json_object_start(response, "result"); + json_for_each_obj(i, result, results) { + json_add_tok(response, json_strdup(tmpctx, reckless->stdoutbuf, result), result+1, reckless->stdoutbuf); + } + json_object_end(response); + + } else { + json_array_start(response, "result"); + json_for_each_arr(i, result, results) { + json_add_string(response, + NULL, + json_strdup(reckless, reckless->stdoutbuf, + result)); + } + json_array_end(response); } - json_array_end(response); json_array_start(response, "log"); logs = json_get_member(reckless->stdoutbuf, toks, "log"); json_for_each_arr(i, log, logs) { @@ -151,6 +162,7 @@ static void reckless_conn_finish(struct io_conn *conn, reckless_result(conn, reckless); /* Don't try to process json if python raised an error. */ } else { + plugin_log(plugin, LOG_DBG, "%s", reckless->stderrbuf); plugin_log(plugin, LOG_DBG, "Reckless process has crashed (%i).", WEXITSTATUS(status)); @@ -211,13 +223,25 @@ static struct io_plan *stderr_conn_init(struct io_conn *conn, return stderr_read_more(conn, reckless); } +static bool is_single_arg_cmd(const char *command) { + if (strcmp(command, "listconfig")) + return true; + if (strcmp(command, "listavailable")) + return true; + if (strcmp(command, "listinstalled")) + return true; + return false; +} + static struct command_result *reckless_call(struct command *cmd, const char *subcommand, const char *target, const char *target2) { - if (!subcommand || !target) - return command_fail(cmd, PLUGIN_ERROR, "invalid reckless call"); + if (!is_single_arg_cmd(subcommand)) { + if (!subcommand || !target) + return command_fail(cmd, PLUGIN_ERROR, "invalid reckless call"); + } char **my_call; my_call = tal_arrz(tmpctx, char *, 0); tal_arr_expand(&my_call, "reckless"); @@ -232,7 +256,8 @@ static struct command_result *reckless_call(struct command *cmd, tal_arr_expand(&my_call, lconfig.config); } tal_arr_expand(&my_call, (char *) subcommand); - tal_arr_expand(&my_call, (char *) target); + if (target) + tal_arr_expand(&my_call, (char *) target); if (target2) tal_arr_expand(&my_call, (char *) target2); tal_arr_expand(&my_call, NULL); @@ -277,7 +302,7 @@ static struct command_result *json_reckless(struct command *cmd, /* Allow check command to evaluate. */ if (!param(cmd, buf, params, p_req("command", param_string, &command), - p_req("target/subcommand", param_string, &target), + p_opt("target/subcommand", param_string, &target), p_opt("target", param_string, &target2), NULL)) return command_param_failed(); From d16714e7f13f839a58cc756d6666e528f621a835 Mon Sep 17 00:00:00 2001 From: Alex Myers Date: Tue, 28 Oct 2025 12:44:03 -0500 Subject: [PATCH 09/56] reckless: Add `listavailable` command to list plugins available to install reckless listavailable sorts through the available sources to find plugins for which we have installers. Changelog-Added: reckless gained the 'listavailable' command to list available plugins from reckless' sources. --- tools/reckless | 54 +++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 53 insertions(+), 1 deletion(-) diff --git a/tools/reckless b/tools/reckless index fab3bc7be54f..05b70db5de31 100755 --- a/tools/reckless +++ b/tools/reckless @@ -2104,6 +2104,54 @@ def listinstalled(): return plugins +def find_plugin_candidates(source: SourceDir, depth=2) -> list: + """Filter through a source and return any candidates that appear to be + installable plugins with the registered installers.""" + candidates = [] + assert isinstance(source, SourceDir) + if not source.contents and not source.prepopulated: + source.populate() + + guess = InstInfo(source.name, source.location, None, source_dir=source) + if guess.get_inst_details(): + candidates.append(source.name) + if depth <= 1: + return candidates + + for c in source.contents: + if not isinstance(c, SourceDir): + continue + candidates.extend(find_plugin_candidates(c, depth=depth-1)) + + return candidates + + +def available_plugins() -> list: + """List installable plugins available from the sources list""" + candidates = [] + # FIXME: update for LoadedSource object + for source in RECKLESS_SOURCES: + if source.type == Source.UNKNOWN: + log.debug(f'confusing source: {source.type}') + continue + # It takes too many API calls to query for installable plugins accurately. + if source.type == Source.GITHUB_REPO and not source.local_clone: + # FIXME: ignoring non-cloned repos for now. + log.debug(f'unable to search {source.original_source} without a local clone of the repository.') + continue + + if source.local_clone: + candidates.extend(find_plugin_candidates(source.local_clone)) + else: + candidates.extend(find_plugin_candidates(source.content)) + + # Order and deduplicate results + candidates = list(set(candidates)) + candidates.sort() + log.info(' '.join(candidates)) + return candidates + + def report_version() -> str: """return reckless version""" log.info(__VERSION__) @@ -2177,6 +2225,10 @@ if __name__ == '__main__': search_cmd.add_argument('targets', type=str, nargs='*') search_cmd.set_defaults(func=search) + available_cmd = cmd1.add_parser('listavailable', help='list plugins available ' + 'from the sources list') + available_cmd.set_defaults(func=available_plugins) + enable_cmd = cmd1.add_parser('enable', help='dynamically enable a plugin ' 'and update config') enable_cmd.add_argument('targets', type=str, nargs='*') @@ -2216,7 +2268,7 @@ if __name__ == '__main__': all_parsers = [parser, install_cmd, uninstall_cmd, search_cmd, enable_cmd, disable_cmd, list_parse, source_add, source_rem, help_cmd, - update, list_cmd] + update, list_cmd, available_cmd] for p in all_parsers: # This default depends on the .lightning directory p.add_argument('-d', '--reckless-dir', action=StoreIdempotent, From ceae10ff2e06346124770ac11949b2acd09b7544 Mon Sep 17 00:00:00 2001 From: Alex Myers Date: Thu, 30 Oct 2025 18:15:21 -0500 Subject: [PATCH 10/56] reckless: handle lack of cloned source in listavailable cmd --- tools/reckless | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/tools/reckless b/tools/reckless index 05b70db5de31..7f9fe8a8ed2a 100755 --- a/tools/reckless +++ b/tools/reckless @@ -707,7 +707,7 @@ def populate_github_repo(url: str) -> list: return contents -def copy_remote_git_source(github_source: InstInfo): +def copy_remote_git_source(github_source: InstInfo, verbose: bool=True): """clone or fetch & checkout a local copy of a remote git repo""" user, repo = Source.get_github_user_repo(github_source.source_loc) if not user or not repo: @@ -726,7 +726,7 @@ def copy_remote_git_source(github_source: InstInfo): # FIXME: pass LoadedSource and check fetch status assert _git_update(github_source.source_loc, local_path) else: - _git_clone(github_source, local_path) + _git_clone(github_source, local_path, verbose) return SourceDir(local_path, srctype=Source.GIT_LOCAL_CLONE) @@ -1215,8 +1215,11 @@ def _source_search(name: str, src: LoadedSource) -> Union[InstInfo, None]: return None -def _git_clone(src: InstInfo, dest: Union[PosixPath, str]) -> bool: - log.info(f'cloning {src.srctype} {src}') +def _git_clone(src: InstInfo, dest: Union[PosixPath, str], verbose: bool=True) -> bool: + if verbose: + log.info(f'cloning {src.srctype} {src}') + else: + log.debug(f'cloning {src.srctype} {src}') if src.srctype == Source.GITHUB_REPO: assert 'github.com' in src.source_loc source = f"{GITHUB_COM}" + src.source_loc.split("github.com")[-1] @@ -2137,8 +2140,17 @@ def available_plugins() -> list: # It takes too many API calls to query for installable plugins accurately. if source.type == Source.GITHUB_REPO and not source.local_clone: # FIXME: ignoring non-cloned repos for now. - log.debug(f'unable to search {source.original_source} without a local clone of the repository.') - continue + log.debug(f'cloning {source.original_source} in order to search') + clone = copy_remote_git_source(InstInfo(None, + source.original_source, + source.original_source, + source_dir=source.content), + verbose=False) + if not clone: + log.warning(f"could not clone github source {source.original_source}") + continue + source.local_clone = clone + source.local_clone.parent_source = source if source.local_clone: candidates.extend(find_plugin_candidates(source.local_clone)) From 6ec09779eced549a1073944da0e493850aea4fc1 Mon Sep 17 00:00:00 2001 From: Alex Myers Date: Thu, 30 Oct 2025 18:27:51 -0500 Subject: [PATCH 11/56] pytest: add reckless listavailable test --- tests/test_reckless.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tests/test_reckless.py b/tests/test_reckless.py index 6dd479d45bbf..7d613267235d 100644 --- a/tests/test_reckless.py +++ b/tests/test_reckless.py @@ -418,3 +418,15 @@ def test_reckless_uv_install(node_factory): assert r.search_stdout('using installer pythonuv') r.check_stderr() + + +def test_reckless_available(node_factory): + """list available plugins""" + n = get_reckless_node(node_factory) + r = reckless([f"--network={NETWORK}", "listavailable", "-v", "--json"], dir=n.lightning_dir) + assert r.returncode == 0 + # All plugins in the default repo should be found and identified as installable. + assert r.search_stdout('testplugfail') + assert r.search_stdout('testplugpass') + assert r.search_stdout('testplugpyproj') + assert r.search_stdout('testpluguv') From 61a77b7cebd4464b4c3a944e2864866bb52199a3 Mon Sep 17 00:00:00 2001 From: Alex Myers Date: Wed, 5 Nov 2025 17:07:16 -0600 Subject: [PATCH 12/56] reckless: add listconfig command Requested by @ShahanaFarooqui while accessing reckless via rpc in order to find out where the plugins are installed and enabled. --- tests/test_reckless.py | 23 +++++++++++++++++---- tools/reckless | 45 ++++++++++++++++++++++++++++++++++++------ 2 files changed, 58 insertions(+), 10 deletions(-) diff --git a/tests/test_reckless.py b/tests/test_reckless.py index 7d613267235d..43212de3b606 100644 --- a/tests/test_reckless.py +++ b/tests/test_reckless.py @@ -3,6 +3,7 @@ from pathlib import PosixPath, Path import socket from pyln.testing.utils import VALGRIND +import json import pytest import os import re @@ -170,16 +171,30 @@ def test_basic_help(): assert r.search_stdout("options:") or r.search_stdout("optional arguments:") -def test_version(): +def test_reckless_version(node_factory): '''Version should be reported without loading config and should advance - with lightningd''' - r = reckless(["-V", "-v", "--json"]) + with lightningd.''' + node = get_reckless_node(node_factory) + r = reckless(["-V", "-v", "--json"], dir=node.lightning_dir) assert r.returncode == 0 - import json json_out = ''.join(r.stdout) with open('.version', 'r') as f: version = f.readlines()[0].strip() assert json.loads(json_out)['result'][0] == version + assert not r.search_stdout('config file not found') + + # reckless listconfig should report the reckless version as well. + NETWORK = os.environ.get('TEST_NETWORK') + if not NETWORK: + NETWORK = 'regtest' + r = reckless(['listconfig', f'--network={NETWORK}', '--json'], + dir=node.lightning_dir) + assert r.returncode == 0 + result = json.loads(''.join(r.stdout))['result'] + assert result['network'] == NETWORK + assert result['reckless_dir'] == str(node.lightning_dir / 'reckless') + assert result['lightning_conf'] == str(node.lightning_dir / NETWORK / 'config') + assert result['version'] == version def test_contextual_help(node_factory): diff --git a/tools/reckless b/tools/reckless index 7f9fe8a8ed2a..48f27e9db0d0 100755 --- a/tools/reckless +++ b/tools/reckless @@ -99,10 +99,13 @@ class Logger: def reply_json(self): """json output to stdout with accumulated result.""" - if len(log.json_output["result"]) == 1 and \ - isinstance(log.json_output["result"][0], list): - # unpack sources output - log.json_output["result"] = log.json_output["result"][0] + if len(log.json_output["result"]) == 1: + if isinstance(log.json_output["result"][0], list): + # unpack sources output + log.json_output["result"] = log.json_output["result"][0] + elif isinstance(log.json_output['result'][0], dict): + # If result is only a single dict, unpack it from the result list + log.json_output['result'] = log.json_output['result'][0] output = json.dumps(log.json_output, indent=3) + '\n' ratelimit_output(output) @@ -846,6 +849,8 @@ class RecklessConfig(Config): ) Config.__init__(self, path=str(path), default_text=default_text) self.reckless_dir = Path(path).parent + # Which lightning config needs to inherit the reckless config? + self.lightning_conf = None class LightningBitcoinConfig(Config): @@ -1863,6 +1868,7 @@ def load_config(reckless_dir: Union[str, None] = None, reckless_abort('Error: could not load or create the network specific lightningd' ' config (default .lightning/bitcoin)') net_conf.editConfigFile(f'include {reckless_conf.conf_fp}', None) + reckless_conf.lightning_conf = network_path return reckless_conf @@ -2164,6 +2170,31 @@ def available_plugins() -> list: return candidates +def listconfig() -> dict: + """Useful for checking options passed through the reckless-rpc.""" + config = {} + + log.info(f'requested lightning config: {LIGHTNING_CONFIG}') + config.update({'requested_lightning_conf': LIGHTNING_CONFIG}) + + log.info(f'lightning config in use: {RECKLESS_CONFIG.lightning_conf}') + config.update({'lightning_conf': str(RECKLESS_CONFIG.lightning_conf)}) + + log.info(f'lightning directory: {LIGHTNING_DIR}') + config.update({'lightning_dir': str(LIGHTNING_DIR)}) + + log.info(f'reckless directory: {RECKLESS_CONFIG.reckless_dir}') + config.update({'reckless_dir': str(RECKLESS_CONFIG.reckless_dir)}) + + log.info(f'network: {NETWORK}') + config.update({'network': NETWORK}) + + log.info(f'reckless version: {__VERSION__}') + config.update({'version': __VERSION__}) + + return config + + def report_version() -> str: """return reckless version""" log.info(__VERSION__) @@ -2178,7 +2209,7 @@ def unpack_json_arg(json_target: str) -> list: return None if isinstance(targets, list): return targets - log.warning(f'input {target_list} is not a json array') + log.warning(f'input {json_target} is not a json array') return None @@ -2277,10 +2308,12 @@ if __name__ == '__main__': parser.add_argument('-V', '--version', action=StoreTrueIdempotent, const=None, help='print version and exit') + listconfig_cmd = cmd1.add_parser('listconfig', help='list options passed to reckless') + listconfig_cmd.set_defaults(func=listconfig) all_parsers = [parser, install_cmd, uninstall_cmd, search_cmd, enable_cmd, disable_cmd, list_parse, source_add, source_rem, help_cmd, - update, list_cmd, available_cmd] + update, list_cmd, available_cmd, listconfig_cmd] for p in all_parsers: # This default depends on the .lightning directory p.add_argument('-d', '--reckless-dir', action=StoreIdempotent, From 39a761560bfe2b2a6e35cf7a5c15d0811315eb01 Mon Sep 17 00:00:00 2001 From: Alex Myers Date: Thu, 13 Nov 2025 18:14:38 -0600 Subject: [PATCH 13/56] pytest: test reckless listconfig via rpc --- tests/test_reckless.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/tests/test_reckless.py b/tests/test_reckless.py index 43212de3b606..4d971825b886 100644 --- a/tests/test_reckless.py +++ b/tests/test_reckless.py @@ -171,7 +171,7 @@ def test_basic_help(): assert r.search_stdout("options:") or r.search_stdout("optional arguments:") -def test_reckless_version(node_factory): +def test_reckless_version_listconfig(node_factory): '''Version should be reported without loading config and should advance with lightningd.''' node = get_reckless_node(node_factory) @@ -196,6 +196,16 @@ def test_reckless_version(node_factory): assert result['lightning_conf'] == str(node.lightning_dir / NETWORK / 'config') assert result['version'] == version + # Now test via reckless-rpc plugin + node.start() + # FIXME: the plugin finds the installed reckless utility rather than the build directory reckless + listconfig = node.rpc.reckless('listconfig') + print(listconfig) + assert listconfig['result']['lightning_dir'] == str(node.lightning_dir) + assert listconfig['result']['lightning_conf'] == str(node.lightning_dir / NETWORK / 'config') + assert listconfig['result']['network'] == NETWORK + assert listconfig['result']['version'] == version + def test_contextual_help(node_factory): n = get_reckless_node(node_factory) From 8e84ba73d65fc3e35f199531735a0219feb08cac Mon Sep 17 00:00:00 2001 From: Alex Myers Date: Thu, 20 Nov 2025 18:41:45 -0600 Subject: [PATCH 14/56] reckless: check for manifest.json in plugin directories The manifest.json provides a short and long description of the plugin, dependencies, and specifies the entrypoint in case it's not named the same as the plugin. changelog-changed: Reckless uses a manifest in the plugin directory to gain additional details about plugin and installation. --- .../lightningd/testplugfail/manifest.json | 7 + .../lightningd/testplugpass/manifest.json | 7 + .../lightningd/testplugpyproj/manifest.json | 7 + .../lightningd/testpluguv/manifest.json | 7 + tools/reckless | 134 +++++++++++++++--- 5 files changed, 141 insertions(+), 21 deletions(-) create mode 100644 tests/data/recklessrepo/lightningd/testplugfail/manifest.json create mode 100644 tests/data/recklessrepo/lightningd/testplugpass/manifest.json create mode 100644 tests/data/recklessrepo/lightningd/testplugpyproj/manifest.json create mode 100644 tests/data/recklessrepo/lightningd/testpluguv/manifest.json diff --git a/tests/data/recklessrepo/lightningd/testplugfail/manifest.json b/tests/data/recklessrepo/lightningd/testplugfail/manifest.json new file mode 100644 index 000000000000..8c6857aaa70e --- /dev/null +++ b/tests/data/recklessrepo/lightningd/testplugfail/manifest.json @@ -0,0 +1,7 @@ +{ + "name": "testplugfail", + "short_description": "a plugin to test reckless installation where the plugin fails to start", + "long_description": "This plugin is one of several used in the reckless blackbox tests.", + "entrypoint": "testplugfail.py", + "requirements": ["python3"] +} diff --git a/tests/data/recklessrepo/lightningd/testplugpass/manifest.json b/tests/data/recklessrepo/lightningd/testplugpass/manifest.json new file mode 100644 index 000000000000..9df31c6d19f0 --- /dev/null +++ b/tests/data/recklessrepo/lightningd/testplugpass/manifest.json @@ -0,0 +1,7 @@ +{ + "name": "testplugpass", + "short_description": "a plugin to test reckless installation", + "long_description": "This plugin is one of several used in the reckless blackbox tests. This one should success in dependenciy installation, and start up when activated in Core Lightning.", + "entrypoint": "testplugpass.py", + "requirements": ["python3"] +} diff --git a/tests/data/recklessrepo/lightningd/testplugpyproj/manifest.json b/tests/data/recklessrepo/lightningd/testplugpyproj/manifest.json new file mode 100644 index 000000000000..0215ca2fae31 --- /dev/null +++ b/tests/data/recklessrepo/lightningd/testplugpyproj/manifest.json @@ -0,0 +1,7 @@ +{ + "name": "testplugpyproj", + "short_description": "a plugin to test reckless installation", + "long_description": "This plugin is one of several used in the reckless blackbox tests. This one should succeed while specifying dependencies in pyproject.toml.", + "entrypoint": "testplugpyproj.py", + "requirements": ["python3"] +} diff --git a/tests/data/recklessrepo/lightningd/testpluguv/manifest.json b/tests/data/recklessrepo/lightningd/testpluguv/manifest.json new file mode 100644 index 000000000000..31f9ce7027cd --- /dev/null +++ b/tests/data/recklessrepo/lightningd/testpluguv/manifest.json @@ -0,0 +1,7 @@ +{ + "name": "testpluguv", + "short_description": "a plugin to test reckless installation using uv", + "long_description": "This plugin is one of several used in the reckless blackbox tests. This one specifies dependencies for uv in the pyproject.toml and has a corresponding uv.lock file.", + "entrypoint": "testpluguv.py", + "requirements": ["python3"] +} diff --git a/tools/reckless b/tools/reckless index 48f27e9db0d0..d70ffdf30417 100755 --- a/tools/reckless +++ b/tools/reckless @@ -233,6 +233,20 @@ def remove_dir(directory: str) -> bool: return False +class GithubRepository(): + """extract the github user account and repository name.""" + def __init__(self, url: str): + assert 'github.com/' in url.lower() + url_parts = Path(str(url).lower().partition('github.com/')[2]).parts + assert len(url_parts) >= 2 + self.user = url_parts[0] + self.name = url_parts[1].removesuffix('.git') + self.url = url + + def __repr__(self): + return '' + + class Source(Enum): DIRECTORY = 1 LOCAL_REPO = 2 @@ -262,12 +276,11 @@ class Source(Enum): @classmethod def get_github_user_repo(cls, source: str) -> (str, str): 'extract a github username and repository name' - if 'github.com/' not in source.lower(): - return None, None - trailing = Path(source.lower().partition('github.com/')[2]).parts - if len(trailing) < 2: + try: + repo = GithubRepository(source) + return repo.user, repo.name + except: return None, None - return trailing[0], trailing[1].removesuffix('.git') class SubmoduleSource: @@ -325,7 +338,7 @@ class SourceDir(): if self.srctype == Source.DIRECTORY: self.contents = populate_local_dir(self.location) elif self.srctype in [Source.LOCAL_REPO, Source.GIT_LOCAL_CLONE]: - self.contents = populate_local_repo(self.location, parent_source=self.parent_source) + self.contents = populate_local_repo(self.location, parent=self, parent_source=self.parent_source) elif self.srctype == Source.GITHUB_REPO: self.contents = populate_github_repo(self.location) else: @@ -351,7 +364,7 @@ class SourceDir(): return None def __repr__(self): - return f"" + return f"" def __eq__(self, compared): if isinstance(compared, str): @@ -576,7 +589,8 @@ def populate_local_repo(path: str, parent=None, parent_source=None) -> list: parentdir.name) else: relative_path = parentdir.name - child = SourceDir(p, srctype=Source.LOCAL_REPO, + child = SourceDir(p, srctype=parent.srctype, + parent_source=parent_source, relative=relative_path) # ls-tree lists every file in the repo with full path. # No need to populate each directory individually. @@ -611,8 +625,13 @@ def populate_local_repo(path: str, parent=None, parent_source=None) -> list: relative_path = filepath elif basedir.relative: relative_path = str(Path(basedir.relative) / filepath) - assert relative_path - submodule_dir = SourceDir(filepath, srctype=Source.LOCAL_REPO, + else: + relative_path = filepath + if parent: + srctype = parent.srctype + else: + srctype = Source.LOCAL_REPO + submodule_dir = SourceDir(filepath, srctype=srctype, relative=relative_path, parent_source=parent_source) populate_local_repo(Path(path) / filepath, parent=submodule_dir, @@ -710,7 +729,7 @@ def populate_github_repo(url: str) -> list: return contents -def copy_remote_git_source(github_source: InstInfo, verbose: bool=True): +def copy_remote_git_source(github_source: InstInfo, verbose: bool=True) -> SourceDir: """clone or fetch & checkout a local copy of a remote git repo""" user, repo = Source.get_github_user_repo(github_source.source_loc) if not user or not repo: @@ -2113,17 +2132,84 @@ def listinstalled(): return plugins -def find_plugin_candidates(source: SourceDir, depth=2) -> list: +def have_files(source: SourceDir): + """Do we have direct access to the files in this directory?""" + if source.srctype in [Source.DIRECTORY, Source.LOCAL_REPO, + Source.GIT_LOCAL_CLONE]: + return True + log.info(f'no files in {source.name} ({source.srctype})') + return False + + +def fetch_manifest(source: SourceDir) -> dict: + """read and ingest a manifest from the provided source.""" + log.debug(f'ingesting manifest from {source.name}: {source.location}/manifest.json ({source.srctype})') + # local_path = RECKLESS_DIR / '.remote_sources' / user + if source.srctype not in [Source.GIT_LOCAL_CLONE, Source.LOCAL_REPO, Source.DIRECTORY]: + log.info(f'oops! {source.srctype}') + return None + if source.srctype == Source.GIT_LOCAL_CLONE: + try: + repo = GithubRepository(source.parent_source.original_source) + path = RECKLESS_DIR / '.remote_sources' / repo.user / repo.name + except AssertionError: + log.info(f'could not parse github source {source.parent_source.original_source}') + return None + elif source.srctype in [Source.DIRECTORY, Source.LOCAL_REPO]: + path = Path(source.location) + else: + raise Exception(f"cannot access manifest in {source.srctype}: {source}") + if source.relative: + path = path / source.relative + path = path / 'manifest.json' + if not path.exists(): + return None + with open(path, 'r+') as manifest_file: + try: + manifest = json.load(manifest_file) + return manifest + except json.decoder.JSONDecodeError: + log.warning(f'{source.name} contains malformed manifest ({source.parent_source.original_source})') + return None + + +def find_plugin_candidates(source: Union[LoadedSource, SourceDir], depth=2) -> list: """Filter through a source and return any candidates that appear to be installable plugins with the registered installers.""" + if isinstance(source, LoadedSource): + if source.local_clone: + return find_plugin_candidates(source.local_clone) + return find_plugin_candidates(source.content) + candidates = [] assert isinstance(source, SourceDir) if not source.contents and not source.prepopulated: source.populate() + for s in source.contents: + if isinstance(s, SourceDir): + assert s.srctype == source.srctype, f'source dir {s.name}, {s.srctype} did not inherit {source.srctype} from {source.name}' + assert s.parent_source == source.parent_source, f'source dir {s.name} did not inherit parent {source.parent_source} from {source.name}' guess = InstInfo(source.name, source.location, None, source_dir=source) + guess.srctype = source.srctype + manifest = None if guess.get_inst_details(): - candidates.append(source.name) + guess.srctype = source.srctype + guess.source_dir.srctype = source.srctype + if guess.source_dir.find('manifest.json'): + # FIXME: Handle github source case + if have_files(guess.source_dir): + manifest = fetch_manifest(guess.source_dir) + + if manifest: + candidate = manifest + else: + candidate = {'name': source.name, + 'short_description': None, + 'long_description': None, + 'entrypoint': guess.entry, + 'requirements': []} + candidates.append(candidate) if depth <= 1: return candidates @@ -2152,21 +2238,27 @@ def available_plugins() -> list: source.original_source, source_dir=source.content), verbose=False) + clone.srctype = Source.GIT_LOCAL_CLONE + clone.parent_source = source if not clone: log.warning(f"could not clone github source {source.original_source}") continue source.local_clone = clone source.local_clone.parent_source = source - if source.local_clone: - candidates.extend(find_plugin_candidates(source.local_clone)) - else: - candidates.extend(find_plugin_candidates(source.content)) + candidates.extend(find_plugin_candidates(source)) + + # json output requested + if log.capture: + return candidates + + for c in candidates: + log.info(c['name']) + if c['short_description']: + log.info(f'\tdescription: {c["short_description"]}') + if c['requirements']: + log.info(f'\trequirements: {c["requirements"]}') - # Order and deduplicate results - candidates = list(set(candidates)) - candidates.sort() - log.info(' '.join(candidates)) return candidates From 15324d2fe205883cc94212a4e9363667283b7b7b Mon Sep 17 00:00:00 2001 From: Alex Myers Date: Thu, 18 Dec 2025 17:31:46 -0600 Subject: [PATCH 15/56] reckless: add shebang installer for fancy plugins This allows plugins with a uv run script to install themselves. Unfortunately it requires file access to all potential entrypoints to check if they are installable. Changelog-Added: reckless can now install plugins executable by shebang. --- .../lightningd/testplugshebang/manifest.json | 7 + .../testplugshebang/requirements.txt | 2 + .../testplugshebang/testplugshebang.py | 27 ++++ .../rkls_api_lightningd_plugins.json | 9 ++ tests/test_reckless.py | 15 +++ tools/reckless | 126 +++++++++++++++++- 6 files changed, 180 insertions(+), 6 deletions(-) create mode 100644 tests/data/recklessrepo/lightningd/testplugshebang/manifest.json create mode 100644 tests/data/recklessrepo/lightningd/testplugshebang/requirements.txt create mode 100755 tests/data/recklessrepo/lightningd/testplugshebang/testplugshebang.py diff --git a/tests/data/recklessrepo/lightningd/testplugshebang/manifest.json b/tests/data/recklessrepo/lightningd/testplugshebang/manifest.json new file mode 100644 index 000000000000..379447fe5ba8 --- /dev/null +++ b/tests/data/recklessrepo/lightningd/testplugshebang/manifest.json @@ -0,0 +1,7 @@ +{ + "name": "testplugshebang", + "short_description": "a plugin to test reckless installation with a UV shebang", + "long_description": "This plugin is used in the reckless blackbox tests. This one manages its own dependency installation with uv invoked by #! from within the plugin.", + "entrypoint": "testplugshebang.py", + "requirements": ["python3"] +} diff --git a/tests/data/recklessrepo/lightningd/testplugshebang/requirements.txt b/tests/data/recklessrepo/lightningd/testplugshebang/requirements.txt new file mode 100644 index 000000000000..7b19e677138d --- /dev/null +++ b/tests/data/recklessrepo/lightningd/testplugshebang/requirements.txt @@ -0,0 +1,2 @@ +pyln-client + diff --git a/tests/data/recklessrepo/lightningd/testplugshebang/testplugshebang.py b/tests/data/recklessrepo/lightningd/testplugshebang/testplugshebang.py new file mode 100755 index 000000000000..13c6a0caa425 --- /dev/null +++ b/tests/data/recklessrepo/lightningd/testplugshebang/testplugshebang.py @@ -0,0 +1,27 @@ +#!/usr/bin/env -S uv run --script + +# /// script +# requires-python = ">=3.9.2" +# dependencies = [ +# "pyln-client>=25.12", +# ] +# /// + +from pyln.client import Plugin + +plugin = Plugin() + +__version__ = 'v1' + + +@plugin.init() +def init(options, configuration, plugin, **kwargs): + plugin.log("testplugshebang initialized") + + +@plugin.method("plugintest") +def plugintest(plugin): + return ("success") + + +plugin.run() diff --git a/tests/data/recklessrepo/rkls_api_lightningd_plugins.json b/tests/data/recklessrepo/rkls_api_lightningd_plugins.json index a91c4844d898..e28e55c853e6 100644 --- a/tests/data/recklessrepo/rkls_api_lightningd_plugins.json +++ b/tests/data/recklessrepo/rkls_api_lightningd_plugins.json @@ -34,5 +34,14 @@ "git_url": "https://api.github.com/repos/lightningd/plugins/git/trees/testplugpyproj", "download_url": null, "type": "dir" + }, + { + "name": "testplugshebang", + "path": "testplugshebang", + "url": "https://api.github.com/repos/lightningd/plugins/contents/webhook?ref=master", + "html_url": "https://github.com/lightningd/plugins/tree/master/testplugshebang", + "git_url": "https://api.github.com/repos/lightningd/plugins/git/trees/testplugshebang", + "download_url": null, + "type": "dir" } ] diff --git a/tests/test_reckless.py b/tests/test_reckless.py index 4d971825b886..858bbee0d50b 100644 --- a/tests/test_reckless.py +++ b/tests/test_reckless.py @@ -445,6 +445,21 @@ def test_reckless_uv_install(node_factory): r.check_stderr() +@unittest.skipIf(VALGRIND, "node too slow for starting plugin under valgrind") +def test_reckless_shebang_install(node_factory): + node = get_reckless_node(node_factory) + node.start() + r = reckless([f"--network={NETWORK}", "-v", "install", "testplugshebang"], + dir=node.lightning_dir) + assert r.returncode == 0 + installed_path = Path(node.lightning_dir) / 'reckless/testplugshebang' + assert installed_path.is_dir() + assert node.rpc.plugintest() == 'success' + + assert r.search_stdout('using installer shebang') + r.check_stderr() + + def test_reckless_available(node_factory): """list available plugins""" n = get_reckless_node(node_factory) diff --git a/tools/reckless b/tools/reckless index d70ffdf30417..4321dac1f535 100755 --- a/tools/reckless +++ b/tools/reckless @@ -157,6 +157,9 @@ class Installer: self.manager = manager # dependency manager (if required) self.dependency_file = None self.dependency_call = None + # extra check routine to see if a source is installable by this Installer + self.check = None + def __repr__(self): return (f' bool: + def installable(self, source) -> bool: '''Validate the necessary compiler and package manager executables are available to install. If these are defined, they are considered mandatory even though the user may have the requisite packages already @@ -184,6 +187,8 @@ class Installer: return False if self.manager and not shutil.which(self.manager): return False + if self.check: + return self.check(source) return True def add_entrypoint(self, entry: str): @@ -484,9 +489,14 @@ class InstInfo: found_entry = sub.find(g, ftype=SourceFile) if found_entry: break - # FIXME: handle a list of dependencies - found_dep = sub.find(inst.dependency_file, - ftype=SourceFile) + + if inst.dependency_file: + # FIXME: handle a list of dependencies + found_dep = sub.find(inst.dependency_file, + ftype=SourceFile) + else: + found_dep = None + if found_entry: # Success! if found_dep: @@ -1031,6 +1041,47 @@ def install_to_python_virtual_environment(cloned_plugin: InstInfo): return cloned_plugin +def have_files(source: SourceDir): + """Do we have direct access to the files in this directory?""" + if source.srctype in [Source.DIRECTORY, Source.LOCAL_REPO, + Source.GIT_LOCAL_CLONE]: + return True + log.info(f'no files in {source.name} ({source.srctype})') + return False + + +def fetch_manifest(source: SourceDir) -> dict: + """read and ingest a manifest from the provided source.""" + log.debug(f'ingesting manifest from {source.name}: {source.location}/manifest.json ({source.srctype})') + # local_path = RECKLESS_DIR / '.remote_sources' / user + if source.srctype not in [Source.GIT_LOCAL_CLONE, Source.LOCAL_REPO, Source.DIRECTORY]: + log.info(f'oops! {source.srctype}') + return None + if source.srctype == Source.GIT_LOCAL_CLONE: + try: + repo = GithubRepository(source.parent_source.original_source) + path = RECKLESS_DIR / '.remote_sources' / repo.user / repo.name + except AssertionError: + log.info(f'could not parse github source {source.parent_source.original_source}') + return None + elif source.srctype in [Source.DIRECTORY, Source.LOCAL_REPO]: + path = Path(source.location) + else: + raise Exception(f"cannot access manifest in {source.srctype}: {source}") + if source.relative: + path = path / source.relative + path = path / 'manifest.json' + if not path.exists(): + return None + with open(path, 'r+') as manifest_file: + try: + manifest = json.load(manifest_file) + return manifest + except json.decoder.JSONDecodeError: + log.warning(f'{source.name} contains malformed manifest ({source.parent_source.original_source})') + return None + + def cargo_installation(cloned_plugin: InstInfo): call = ['cargo', 'build', '--release', '-vv'] # FIXME: the symlinked Cargo.toml allows the installer to identify a valid @@ -1144,6 +1195,45 @@ def install_python_uv_legacy(cloned_plugin: InstInfo): return cloned_plugin +def open_source_entrypoint(source: InstInfo) -> str: + if source.srctype not in [Source.GIT_LOCAL_CLONE, Source.LOCAL_REPO, Source.DIRECTORY]: + log.info(f'oops! {source.srctype}') + return None + assert source.entry + file = Path(source.source_loc) + # if source.subdir: + # file /= source.subdir + file /= source.entry + log.debug(f'checking entry file {str(file)}') + if file.exists(): + # FIXME: check file encoding + try: + with open(file, 'r') as f: + return f.read() + except UnicodeDecodeError: + log.debug('failed to read source file') + return None + else: + log.debug('could not find source file') + + return None + +def check_for_shebang(source: InstInfo) -> bool: + log.debug(f'checking for shebang in {source}') + if source.source_dir: + source.get_inst_details() + if have_files(source.source_dir): + entrypoint_file = open_source_entrypoint(source) + if entrypoint_file.split('\n')[0].startswith('#!'): + # Calling the python interpreter will not manage dependencies. + # Leave this to another python installer. + for interpreter in ['bin/python', 'env python']: + if interpreter in entrypoint_file.split('\n')[0]: + return False + return True + return False + + python3venv = Installer('python3venv', exe='python3', manager='pip', entry='{name}.py') python3venv.add_entrypoint('{name}') @@ -1186,7 +1276,12 @@ rust_cargo = Installer('rust', manager='cargo', entry='Cargo.toml') rust_cargo.add_dependency_file('Cargo.toml') rust_cargo.dependency_call = cargo_installation -INSTALLERS = [pythonuv, pythonuvlegacy, python3venv, poetryvenv, +shebang = Installer('shebang', entry='{name}.py') +shebang.add_entrypoint('{name}') +# An extra installable check to see if a #! is present in the file +shebang.check = check_for_shebang + +INSTALLERS = [shebang, pythonuv, pythonuvlegacy, python3venv, poetryvenv, pyprojectViaPip, nodejs, rust_cargo] @@ -1376,6 +1471,8 @@ def _checkout_commit(orig_src: InstInfo, def _install_plugin(src: InstInfo) -> Union[InstInfo, None]: """make sure the repo exists and clone it.""" log.debug(f'Install requested from {src}.') + if src.source_dir and src.source_dir.parent_source: + log.debug(f'source has parent {src.source_dir.parent_source}') if RECKLESS_CONFIG is None: log.error('reckless install directory unavailable') return None @@ -1411,6 +1508,8 @@ def _install_plugin(src: InstInfo) -> Union[InstInfo, None]: # FIXME: Validate path was cloned successfully. # Depending on how we accessed the original source, there may be install # details missing. Searching the cloned repo makes sure we have it. + # FIXME: This could be cloned to .remotesources and the global sources + # could then be updated with this new LoadedSource to save on additional cloning. clone = LoadedSource(plugin_path) clone.content.populate() # Make sure we don't try to fetch again! @@ -1426,14 +1525,29 @@ def _install_plugin(src: InstInfo) -> Union[InstInfo, None]: if not plugin_path: return None + # FIXME: replace src wholesale + # We have a hunch it's in this directory/source, so link it here. + inst_check_src = copy.copy(src) + if not inst_check_src.source_dir: + inst_check_src.source_loc = plugin_path + inst_check_src.source_dir = clone.content + inst_check_src.source_dir.parent_source = clone + + if src.srctype == Source.GITHUB_REPO: + inst_check_src.srctype = Source.GIT_LOCAL_CLONE + else: + inst_check_src.srctype = clone.type + # Find a suitable installer INSTALLER = None for inst_method in INSTALLERS: - if not (inst_method.installable() and inst_method.executable()): + if not (inst_method.installable(inst_check_src) and inst_method.executable()): continue if inst_method.dependency_file is not None: if inst_method.dependency_file not in os.listdir(plugin_path): continue + if inst_method.check and not inst_method.check(inst_check_src): + continue log.debug(f"using installer {inst_method.name}") INSTALLER = inst_method break From 0041d7f07c94673fbdcb12c04f62a721d4ab8d6f Mon Sep 17 00:00:00 2001 From: Alex Myers Date: Tue, 6 Jan 2026 11:59:38 -0600 Subject: [PATCH 16/56] reckless install: check and warn if a plugin is already installed. --- tests/test_reckless.py | 6 ++++++ tools/reckless | 26 ++++++++++++++++++++++---- 2 files changed, 28 insertions(+), 4 deletions(-) diff --git a/tests/test_reckless.py b/tests/test_reckless.py index 858bbee0d50b..f69ff213b941 100644 --- a/tests/test_reckless.py +++ b/tests/test_reckless.py @@ -300,6 +300,12 @@ def test_install(node_factory): print(plugin_path) assert os.path.exists(plugin_path) + # Try to install again - should result in a warning. + r = reckless([f"--network={NETWORK}", "-v", "install", "testplugpass"], dir=n.lightning_dir) + r.check_stderr() + assert r.search_stdout('already installed') + assert r.returncode == 0 + def test_install_cleanup(node_factory): """test failed installation and post install cleanup""" diff --git a/tools/reckless b/tools/reckless index 4321dac1f535..349831bedef1 100755 --- a/tools/reckless +++ b/tools/reckless @@ -1731,6 +1731,20 @@ def install(plugin_name: str) -> Union[str, None]: LAST_FOUND = None return None + # Check if we already have this installed. + destination = Path(RECKLESS_CONFIG.reckless_dir) / name.lower() + + if Path(destination).exists(): + # should we run listinstalled first and see what's in the list? + installed = listinstalled(plugin_name) + if installed: + log.info(f'already installed: {list(installed.keys())[0]} in {str(destination)}') + return name + else: + log.warning(f'destination directory {destination} already exists.') + return None + + try: installed = _install_plugin(src) except FileExistsError as err: @@ -2189,8 +2203,9 @@ def extract_metadata(plugin_name: str) -> dict: return metadata -def listinstalled(): - """list all plugins currently managed by reckless""" +def listinstalled(name: str = None): + """list all plugins currently managed by reckless. Optionally passed + a plugin name.""" dir_contents = os.listdir(RECKLESS_CONFIG.reckless_dir) plugins = {} for plugin in dir_contents: @@ -2198,6 +2213,8 @@ def listinstalled(): # skip hidden dirs such as reckless' .remote_sources if plugin[0] == '.': continue + if name and name != plugin: + continue plugins.update({plugin: None}) # Format output in a simple table @@ -2230,8 +2247,9 @@ def listinstalled(): status = "disabled" else: print(f'cant handle {line}') - log.info(f"{plugin:<{name_len}} {md['installed commit']:<{inst_len}} " - f"{md['installation date']:<11} {status}") + if not name: + log.info(f"{plugin:<{name_len}} {md['installed commit']:<{inst_len}} " + f"{md['installation date']:<11} {status}") # This doesn't originate from the metadata, but we want to provide enabled status for json output md['enabled'] = status == "enabled" md['entrypoint'] = installed.entry From d00660a13226b8341af288ec0cd7b61c75e5d35f Mon Sep 17 00:00:00 2001 From: Alex Myers Date: Thu, 8 Jan 2026 13:32:48 -0600 Subject: [PATCH 17/56] reckless: remove github API access The shebang installer requires introspection of files, and the listavailable command reads the manifest for each plugin. Due to the need for file access, cloning all remote repositories is now simpler and faster, so it's time to rip out the github API access code. --- tools/reckless | 184 ++++++++++++++----------------------------------- 1 file changed, 50 insertions(+), 134 deletions(-) diff --git a/tools/reckless b/tools/reckless index 349831bedef1..b4790dec521d 100755 --- a/tools/reckless +++ b/tools/reckless @@ -255,7 +255,7 @@ class GithubRepository(): class Source(Enum): DIRECTORY = 1 LOCAL_REPO = 2 - GITHUB_REPO = 3 + REMOTE_GIT_REPO = 3 OTHER_URL = 4 UNKNOWN = 5 # Cloned from remote source before searching (rather than github API) @@ -309,11 +309,14 @@ class LoadedSource: self.content = SourceDir(source, self.type) self.local_clone = None self.local_clone_fetched = False - if self.type == Source.GITHUB_REPO: + if self.type == Source.REMOTE_GIT_REPO: local = _get_local_clone(source) if local: self.local_clone = SourceDir(local, Source.GIT_LOCAL_CLONE) - self.local_clone.parent_source = self + else: + self.local_clone = copy_remote_git_source(InstInfo(None, source)) + self.content = self.local_clone + self.local_clone.parent_source = self def __repr__(self): return f'' @@ -344,8 +347,9 @@ class SourceDir(): self.contents = populate_local_dir(self.location) elif self.srctype in [Source.LOCAL_REPO, Source.GIT_LOCAL_CLONE]: self.contents = populate_local_repo(self.location, parent=self, parent_source=self.parent_source) - elif self.srctype == Source.GITHUB_REPO: - self.contents = populate_github_repo(self.location) + elif self.srctype == Source.REMOTE_GIT_REPO: + self.contents = copy_remote_git_source(InstInfo(self.name, self.location)).contents + else: raise Exception("populate method undefined for {self.srctype}") # Ensure the relative path of the contents is inherited. @@ -398,11 +402,10 @@ class SourceFile(): class InstInfo: - def __init__(self, name: str, location: str, git_url: str, source_dir: SourceDir=None): + def __init__(self, name: str, location: str, source_dir: SourceDir=None): self.name = name self.source_loc = str(location) # Used for 'git clone' self.source_dir = source_dir # Use this insead of source_loc to only fetch once. - self.git_url: str = git_url # API access for github repos self.srctype: Source = Source.get_type(location) self.entry: SourceFile = None # relative to source_loc or subdir self.deps: str = None @@ -410,7 +413,7 @@ class InstInfo: self.commit: str = None def __repr__(self): - return (f'InstInfo({self.name}, {self.source_loc}, {self.git_url}, ' + return (f'InstInfo({self.name}, {self.source_loc}, ' f'{self.entry}, {self.deps}, {self.subdir})') def get_repo_commit(self) -> Union[str, None]: @@ -422,26 +425,9 @@ class InstInfo: return None return git.stdout.splitlines()[0] - if self.srctype == Source.GITHUB_REPO: - parsed_url = urlparse(self.source_loc) - if 'github.com' not in parsed_url.netloc: - return None - if len(parsed_url.path.split('/')) < 2: - return None - start = 1 - # Maybe we were passed an api.github.com/repo/ url - if 'api' in parsed_url.netloc: - start += 1 - repo_user = parsed_url.path.split('/')[start] - repo_name = parsed_url.path.split('/')[start + 1] - api_url = f'{API_GITHUB_COM}/repos/{repo_user}/{repo_name}/commits?ref=HEAD' - r = urlopen(api_url, timeout=5) - if r.status != 200: - return None - try: - return json.loads(r.read().decode())['0']['sha'] - except: - return None + if self.srctype == Source.REMOTE_GIT_REPO: + # The remote git source is not accessed directly. Use the local clone. + assert False def get_inst_details(self, permissive: bool=False) -> bool: """Search the source_loc for plugin install details. @@ -461,7 +447,7 @@ class InstInfo: if self.srctype in [Source.DIRECTORY, Source.LOCAL_REPO, Source.GIT_LOCAL_CLONE]: depth = 5 - elif self.srctype == Source.GITHUB_REPO: + elif self.srctype == Source.REMOTE_GIT_REPO: depth = 1 def search_dir(self, sub: SourceDir, subdir: bool, @@ -522,7 +508,7 @@ class InstInfo: # Fall back to cloning and searching the local copy instead. except HTTPError: result = None - if self.srctype == Source.GITHUB_REPO: + if self.srctype == Source.REMOTE_GIT_REPO: # clone source to reckless dir target = copy_remote_git_source(self) if not target: @@ -654,91 +640,6 @@ def populate_local_repo(path: str, parent=None, parent_source=None) -> list: return basedir.contents -def source_element_from_repo_api(member: dict): - # api accessed via /contents/ - if 'type' in member and 'name' in member and 'git_url' in member: - if member['type'] == 'dir': - return SourceDir(member['git_url'], srctype=Source.GITHUB_REPO, - name=member['name']) - elif member['type'] == 'file': - # Likely a submodule - if member['size'] == 0: - return SourceDir(None, srctype=Source.GITHUB_REPO, - name=member['name']) - return SourceFile(member['name']) - elif member['type'] == 'commit': - # No path is given by the api here - return SourceDir(None, srctype=Source.GITHUB_REPO, - name=member['name']) - # git_url with /tree/ presents results a little differently - elif 'type' in member and 'path' in member and 'url' in member: - if member['type'] not in ['tree', 'blob']: - log.debug(f' skipping {member["path"]} type={member["type"]}') - if member['type'] == 'tree': - return SourceDir(member['url'], srctype=Source.GITHUB_REPO, - name=member['path']) - elif member['type'] == 'blob': - # This can be a submodule - if member['size'] == 0: - return SourceDir(member['git_url'], srctype=Source.GITHUB_REPO, - name=member['name']) - return SourceFile(member['path']) - elif member['type'] == 'commit': - # No path is given by the api here - return SourceDir(None, srctype=Source.GITHUB_REPO, - name=member['name']) - return None - - -def populate_github_repo(url: str) -> list: - """populate one level of a github repository via REST API""" - # Forces search to clone remote repos (for blackbox testing) - if GITHUB_API_FALLBACK: - with tempfile.NamedTemporaryFile() as tmp: - raise HTTPError(url, 403, 'simulated ratelimit', {}, tmp) - # FIXME: This probably contains leftover cruft. - repo = url.split('/') - while '' in repo: - repo.remove('') - repo_name = None - parsed_url = urlparse(url.removesuffix('.git')) - if 'github.com' not in parsed_url.netloc: - return None - if len(parsed_url.path.split('/')) < 2: - return None - start = 1 - # Maybe we were passed an api.github.com/repo/ url - if 'api' in parsed_url.netloc: - start += 1 - repo_user = parsed_url.path.split('/')[start] - repo_name = parsed_url.path.split('/')[start + 1] - - # Get details from the github API. - if API_GITHUB_COM in url: - api_url = url - else: - api_url = f'{API_GITHUB_COM}/repos/{repo_user}/{repo_name}/contents/' - - git_url = api_url - if "api.github.com" in git_url: - # This lets us redirect to handle blackbox testing - log.debug(f'fetching from gh API: {git_url}') - git_url = (API_GITHUB_COM + git_url.split("api.github.com")[-1]) - # Ratelimiting occurs for non-authenticated GH API calls at 60 in 1 hour. - r = urlopen(git_url, timeout=5) - if r.status != 200: - return False - if 'git/tree' in git_url: - tree = json.loads(r.read().decode())['tree'] - else: - tree = json.loads(r.read().decode()) - contents = [] - for sub in tree: - if source_element_from_repo_api(sub): - contents.append(source_element_from_repo_api(sub)) - return contents - - def copy_remote_git_source(github_source: InstInfo, verbose: bool=True) -> SourceDir: """clone or fetch & checkout a local copy of a remote git repo""" user, repo = Source.get_github_user_repo(github_source.source_loc) @@ -1314,11 +1215,11 @@ def _source_search(name: str, src: LoadedSource) -> Union[InstInfo, None]: """Identify source type, retrieve contents, and populate InstInfo if the relevant contents are found.""" root_dir = src.content - source = InstInfo(name, root_dir.location, None) + source = InstInfo(name, root_dir.location) # If a local clone of a github source already exists, prefer searching # that instead of accessing the github API. - if src.type == Source.GITHUB_REPO: + if src.type == Source.REMOTE_GIT_REPO: if src.local_clone: if not src.local_clone_fetched: # FIXME: Pass the LoadedSource here? @@ -1326,10 +1227,18 @@ def _source_search(name: str, src: LoadedSource) -> Union[InstInfo, None]: src.local_clone_fetched = True log.debug(f'fetching local clone of {src.original_source}') log.debug(f"Using local clone of {src}: {src.local_clone.location}") + + # FIXME: ideally, the InstInfo object would have a concept of the + # original LoadedSource and get_inst_details would follow the local clone source.source_loc = str(src.local_clone.location) source.srctype = Source.GIT_LOCAL_CLONE if source.get_inst_details(permissive=True): + # If we have a local clone, report back the original location and type, + # not the clone that was traversed. + if source.srctype is Source.GIT_LOCAL_CLONE: + source.source_loc = src.original_source + source.srctype = src.type return source return None @@ -1339,9 +1248,11 @@ def _git_clone(src: InstInfo, dest: Union[PosixPath, str], verbose: bool=True) - log.info(f'cloning {src.srctype} {src}') else: log.debug(f'cloning {src.srctype} {src}') - if src.srctype == Source.GITHUB_REPO: - assert 'github.com' in src.source_loc - source = f"{GITHUB_COM}" + src.source_loc.split("github.com")[-1] + if src.srctype == Source.REMOTE_GIT_REPO: + if 'github.com' in src.source_loc: + source = f"{GITHUB_COM}" + src.source_loc.split("github.com")[-1] + else: + source = src.source_loc elif src.srctype in [Source.LOCAL_REPO, Source.OTHER_URL, Source.GIT_LOCAL_CLONE]: source = src.source_loc @@ -1360,8 +1271,14 @@ def _git_clone(src: InstInfo, dest: Union[PosixPath, str], verbose: bool=True) - def _git_update(github_source: str, local_copy: PosixPath): + + if 'github.com' in github_source: + source = GITHUB_COM + github_source.split('github.com')[-1] + else: + source = github_source + # Ensure this is the correct source - git = run(['git', 'remote', 'set-url', 'origin', github_source], + git = run(['git', 'remote', 'set-url', 'origin', source], cwd=str(local_copy), stdout=PIPE, stderr=PIPE, text=True, check=False, timeout=60) assert git.returncode == 0 @@ -1386,7 +1303,7 @@ def _git_update(github_source: str, local_copy: PosixPath): default_branch = git.stdout.splitlines()[0] if default_branch not in ['origin/master', 'origin/main']: log.debug(f'UNUSUAL: fetched default branch {default_branch} for ' - f'{github_source}') + f'{source}') # Checkout default branch git = run(['git', 'checkout', default_branch], @@ -1433,7 +1350,7 @@ def _checkout_commit(orig_src: InstInfo, cloned_src: InstInfo, cloned_path: PosixPath): # Check out and verify commit/tag if source was a repository - if orig_src.srctype in [Source.LOCAL_REPO, Source.GITHUB_REPO, + if orig_src.srctype in [Source.LOCAL_REPO, Source.REMOTE_GIT_REPO, Source.OTHER_URL, Source.GIT_LOCAL_CLONE]: if orig_src.commit: log.debug(f"Checking out {orig_src.commit}") @@ -1500,7 +1417,7 @@ def _install_plugin(src: InstInfo) -> Union[InstInfo, None]: f" {full_source_path}")) create_dir(clone_path) shutil.copytree(full_source_path, plugin_path) - elif src.srctype in [Source.LOCAL_REPO, Source.GITHUB_REPO, + elif src.srctype in [Source.LOCAL_REPO, Source.REMOTE_GIT_REPO, Source.OTHER_URL, Source.GIT_LOCAL_CLONE]: # clone git repository to /tmp/reckless-... if not _git_clone(src, plugin_path): @@ -1533,7 +1450,7 @@ def _install_plugin(src: InstInfo) -> Union[InstInfo, None]: inst_check_src.source_dir = clone.content inst_check_src.source_dir.parent_source = clone - if src.srctype == Source.GITHUB_REPO: + if src.srctype == Source.REMOTE_GIT_REPO: inst_check_src.srctype = Source.GIT_LOCAL_CLONE else: inst_check_src.srctype = clone.type @@ -1712,7 +1629,7 @@ def install(plugin_name: str) -> Union[str, None]: src = None if direct_location: log.debug(f"install of {name} requested from {direct_location}") - src = InstInfo(name, direct_location, name) + src = InstInfo(name, direct_location) # Treating a local git repo as a directory allows testing # uncommitted changes. if src and src.srctype == Source.LOCAL_REPO: @@ -1764,7 +1681,7 @@ def install(plugin_name: str) -> Union[str, None]: def uninstall(plugin_name: str) -> str: - """dDisables plugin and deletes the plugin's reckless dir. Returns the + """Disables plugin and deletes the plugin's reckless dir. Returns the status of the uninstall attempt.""" assert isinstance(plugin_name, str) log.debug(f'Uninstalling plugin {plugin_name}') @@ -1818,7 +1735,7 @@ def search(plugin_name: str) -> Union[InstInfo, None]: for src in RECKLESS_SOURCES: # Search repos named after the plugin before collections - if src.type == Source.GITHUB_REPO: + if src.type == Source.REMOTE_GIT_REPO: if src.original_source.split('/')[-1].lower().removesuffix('.git') == plugin_name.lower(): ordered_sources.remove(src) ordered_sources.insert(0, src) @@ -1848,7 +1765,7 @@ def search(plugin_name: str) -> Union[InstInfo, None]: log.debug(f'cannot search {source.type} {source.original_source}') continue if source.type in [Source.DIRECTORY, Source.LOCAL_REPO, - Source.GITHUB_REPO, Source.OTHER_URL]: + Source.REMOTE_GIT_REPO, Source.OTHER_URL]: found = _source_search(plugin_name, source) if found: log.debug(f"{found}, {found.srctype}") @@ -2130,7 +2047,7 @@ def update_plugin(plugin_name: str) -> tuple: return (None, UpdateStatus.REFUSING_UPDATE) src = InstInfo(plugin_name, - metadata['original source'], None) + metadata['original source']) if not src.get_inst_details(): log.error(f'cannot locate {plugin_name} in original source {metadata["original_source"]}') return (None, UpdateStatus.ERROR) @@ -2322,7 +2239,7 @@ def find_plugin_candidates(source: Union[LoadedSource, SourceDir], depth=2) -> l assert s.srctype == source.srctype, f'source dir {s.name}, {s.srctype} did not inherit {source.srctype} from {source.name}' assert s.parent_source == source.parent_source, f'source dir {s.name} did not inherit parent {source.parent_source} from {source.name}' - guess = InstInfo(source.name, source.location, None, source_dir=source) + guess = InstInfo(source.name, source.location, source_dir=source) guess.srctype = source.srctype manifest = None if guess.get_inst_details(): @@ -2362,11 +2279,10 @@ def available_plugins() -> list: log.debug(f'confusing source: {source.type}') continue # It takes too many API calls to query for installable plugins accurately. - if source.type == Source.GITHUB_REPO and not source.local_clone: + if source.type == Source.REMOTE_GIT_REPO and not source.local_clone: # FIXME: ignoring non-cloned repos for now. log.debug(f'cloning {source.original_source} in order to search') clone = copy_remote_git_source(InstInfo(None, - source.original_source, source.original_source, source_dir=source.content), verbose=False) @@ -2608,7 +2524,6 @@ if __name__ == '__main__': LIGHTNING_CONFIG = args.conf RECKLESS_CONFIG = load_config(reckless_dir=str(RECKLESS_DIR), network=NETWORK) - RECKLESS_SOURCES = load_sources() API_GITHUB_COM = 'https://api.github.com' GITHUB_COM = 'https://github.com' # Used for blackbox testing to avoid hitting github servers @@ -2621,6 +2536,7 @@ if __name__ == '__main__': if 'GITHUB_API_FALLBACK' in os.environ: GITHUB_API_FALLBACK = os.environ['GITHUB_API_FALLBACK'] + RECKLESS_SOURCES = load_sources() if 'targets' in args: # and len(args.targets) > 0: if args.func.__name__ == 'help_alias': log.add_result(args.func(args.targets)) From 858ada0819448b7aeccddff480dee62bd9b216f5 Mon Sep 17 00:00:00 2001 From: Alex Myers Date: Thu, 8 Jan 2026 14:03:25 -0600 Subject: [PATCH 18/56] reckless: remove remaining traces of API access Including the canned server in the pytest infrastructure. --- .../rkls_api_lightningd_plugins.json | 47 ------------------- tests/test_reckless.py | 27 +++-------- tools/reckless | 46 +++--------------- 3 files changed, 12 insertions(+), 108 deletions(-) delete mode 100644 tests/data/recklessrepo/rkls_api_lightningd_plugins.json diff --git a/tests/data/recklessrepo/rkls_api_lightningd_plugins.json b/tests/data/recklessrepo/rkls_api_lightningd_plugins.json deleted file mode 100644 index e28e55c853e6..000000000000 --- a/tests/data/recklessrepo/rkls_api_lightningd_plugins.json +++ /dev/null @@ -1,47 +0,0 @@ -[ - { - "name": "testplugpass", - "path": "testplugpass", - "url": "https://api.github.com/repos/lightningd/plugins/contents/webhook?ref=master", - "html_url": "https://github.com/lightningd/plugins/tree/master/testplugpass", - "git_url": "https://api.github.com/repos/lightningd/plugins/git/trees/testplugpass", - "download_url": null, - "type": "dir" - }, - { - "name": "testpluguv", - "path": "testpluguv", - "url": "https://api.github.com/repos/lightningd/plugins/contents/webhook?ref=master", - "html_url": "https://github.com/lightningd/plugins/tree/master/testpluguv", - "git_url": "https://api.github.com/repos/lightningd/plugins/git/trees/testpluguv", - "download_url": null, - "type": "dir" - }, - { - "name": "testplugfail", - "path": "testplugfail", - "url": "https://api.github.com/repos/lightningd/plugins/contents/testplugfail?ref=master", - "html_url": "https://github.com/lightningd/plugins/tree/master/testplugfail", - "git_url": "https://api.github.com/repos/lightningd/plugins/git/trees/testplugfail", - "download_url": null, - "type": "dir" - }, - { - "name": "testplugpyproj", - "path": "testplugpyproj", - "url": "https://api.github.com/repos/lightningd/plugins/contents/webhook?ref=master", - "html_url": "https://github.com/lightningd/plugins/tree/master/testplugpyproj", - "git_url": "https://api.github.com/repos/lightningd/plugins/git/trees/testplugpyproj", - "download_url": null, - "type": "dir" - }, - { - "name": "testplugshebang", - "path": "testplugshebang", - "url": "https://api.github.com/repos/lightningd/plugins/contents/webhook?ref=master", - "html_url": "https://github.com/lightningd/plugins/tree/master/testplugshebang", - "git_url": "https://api.github.com/repos/lightningd/plugins/git/trees/testplugshebang", - "download_url": null, - "type": "dir" - } -] diff --git a/tests/test_reckless.py b/tests/test_reckless.py index f69ff213b941..3bb1fde220f0 100644 --- a/tests/test_reckless.py +++ b/tests/test_reckless.py @@ -1,15 +1,13 @@ -from fixtures import * # noqa: F401,F403 -import subprocess -from pathlib import PosixPath, Path -import socket -from pyln.testing.utils import VALGRIND import json -import pytest import os +from pathlib import PosixPath, Path import re -import shutil +import subprocess import time import unittest +from fixtures import * # noqa: F401,F403 +from pyln.testing.utils import VALGRIND +import pytest @pytest.fixture(autouse=True) @@ -22,20 +20,10 @@ def canned_github_server(directory): if os.environ.get('LIGHTNING_CLI') is None: os.environ['LIGHTNING_CLI'] = str(FILE_PATH.parent / 'cli/lightning-cli') print('LIGHTNING_CALL: ', os.environ.get('LIGHTNING_CLI')) - # Use socket to provision a random free port - sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - sock.bind(('localhost', 0)) - free_port = str(sock.getsockname()[1]) - sock.close() global my_env my_env = os.environ.copy() - # This tells reckless to redirect to the canned server rather than github. - my_env['REDIR_GITHUB_API'] = f'http://127.0.0.1:{free_port}/api' + # This tells reckless to redirect to the local test plugins repo rather than github. my_env['REDIR_GITHUB'] = directory - my_env['FLASK_RUN_PORT'] = free_port - my_env['FLASK_APP'] = str(FILE_PATH / 'rkls_github_canned_server') - server = subprocess.Popen(["python3", "-m", "flask", "run"], - env=my_env) # Generate test plugin repository to test reckless against. repo_dir = os.path.join(directory, "lightningd") @@ -84,13 +72,10 @@ def canned_github_server(directory): del my_env['GIT_DIR'] del my_env['GIT_WORK_TREE'] del my_env['GIT_INDEX_FILE'] - # We also need the github api data for the repo which will be served via http - shutil.copyfile(str(FILE_PATH / 'data/recklessrepo/rkls_api_lightningd_plugins.json'), os.path.join(directory, 'rkls_api_lightningd_plugins.json')) yield # Delete requirements.txt from the testplugpass directory with open(requirements_file_path, 'w') as f: f.write(f"pyln-client\n\n") - server.terminate() class RecklessResult: diff --git a/tools/reckless b/tools/reckless index b4790dec521d..7ff79b690555 100755 --- a/tools/reckless +++ b/tools/reckless @@ -18,7 +18,6 @@ import types from typing import Union from urllib.parse import urlparse from urllib.request import urlopen -from urllib.error import HTTPError import venv @@ -258,7 +257,7 @@ class Source(Enum): REMOTE_GIT_REPO = 3 OTHER_URL = 4 UNKNOWN = 5 - # Cloned from remote source before searching (rather than github API) + # Cloned from remote source before searching GIT_LOCAL_CLONE = 6 @classmethod @@ -302,7 +301,7 @@ class SubmoduleSource: class LoadedSource: """Allows loading all sources only once per call of reckless. Initialized with a single line of the reckless .sources file. Keeping state also allows - minimizing API calls and refetching repositories.""" + minimizing refetching repositories.""" def __init__(self, source: str): self.original_source = source self.type = Source.get_type(source) @@ -445,16 +444,12 @@ class InstInfo: # Set recursion for how many directories deep we should search depth = 0 if self.srctype in [Source.DIRECTORY, Source.LOCAL_REPO, - Source.GIT_LOCAL_CLONE]: + Source.GIT_LOCAL_CLONE, Source.REMOTE_GIT_REPO]: depth = 5 - elif self.srctype == Source.REMOTE_GIT_REPO: - depth = 1 def search_dir(self, sub: SourceDir, subdir: bool, recursion: int) -> Union[SourceDir, None]: assert isinstance(recursion, int) - # carveout for archived plugins in lightningd/plugins. Other repos - # are only searched by API at the top level. if recursion == 0 and 'archive' in sub.name.lower(): pass # If unable to search deeper, resort to matching directory name @@ -501,27 +496,7 @@ class InstInfo: return success return None - try: - result = search_dir(self, target, False, depth) - # Using the rest API of github.com may result in a - # "Error 403: rate limit exceeded" or other access issues. - # Fall back to cloning and searching the local copy instead. - except HTTPError: - result = None - if self.srctype == Source.REMOTE_GIT_REPO: - # clone source to reckless dir - target = copy_remote_git_source(self) - if not target: - log.warning(f"could not clone github source {self}") - return False - log.debug(f"falling back to cloning remote repo {self}") - # Update to reflect use of a local clone - self.source_loc = str(target.location) - self.srctype = target.srctype - result = search_dir(self, target, False, 5) - - if not result: - return False + result = search_dir(self, target, False, depth) if result: if result != target: @@ -1202,8 +1177,7 @@ def help_alias(targets: list): def _get_local_clone(source: str) -> Union[Path, None]: - """Returns the path of a local repository clone of a github source. If one - already exists, prefer searching that to accessing the github API.""" + """Returns the path of a local repository clone of a github source.""" user, repo = Source.get_github_user_repo(source) local_clone_location = RECKLESS_DIR / '.remote_sources' / user / repo if local_clone_location.exists(): @@ -1217,8 +1191,7 @@ def _source_search(name: str, src: LoadedSource) -> Union[InstInfo, None]: root_dir = src.content source = InstInfo(name, root_dir.location) - # If a local clone of a github source already exists, prefer searching - # that instead of accessing the github API. + # Remote git sources require a local clone before searching. if src.type == Source.REMOTE_GIT_REPO: if src.local_clone: if not src.local_clone_fetched: @@ -2524,18 +2497,11 @@ if __name__ == '__main__': LIGHTNING_CONFIG = args.conf RECKLESS_CONFIG = load_config(reckless_dir=str(RECKLESS_DIR), network=NETWORK) - API_GITHUB_COM = 'https://api.github.com' GITHUB_COM = 'https://github.com' # Used for blackbox testing to avoid hitting github servers - if 'REDIR_GITHUB_API' in os.environ: - API_GITHUB_COM = os.environ['REDIR_GITHUB_API'] if 'REDIR_GITHUB' in os.environ: GITHUB_COM = os.environ['REDIR_GITHUB'] - GITHUB_API_FALLBACK = False - if 'GITHUB_API_FALLBACK' in os.environ: - GITHUB_API_FALLBACK = os.environ['GITHUB_API_FALLBACK'] - RECKLESS_SOURCES = load_sources() if 'targets' in args: # and len(args.targets) > 0: if args.func.__name__ == 'help_alias': From 3a404553173b384a579763422186ecd7dea4b890 Mon Sep 17 00:00:00 2001 From: Alex Myers Date: Thu, 15 Jan 2026 11:14:13 -0600 Subject: [PATCH 19/56] reckless: avoid populating uninitialized repo submodules with higher level repository contents. --- tools/reckless | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/tools/reckless b/tools/reckless index 7ff79b690555..3f7f56c05f71 100755 --- a/tools/reckless +++ b/tools/reckless @@ -579,11 +579,14 @@ def populate_local_repo(path: str, parent=None, parent_source=None) -> list: return None submodules = [] for sub in proc.stdout.splitlines(): + # `git submodule status` can list higher level directory contents. + if sub.split()[1].startswith('..') or sub.split()[1].startswith('./'): + continue submodules.append(sub.split()[1]) # FIXME: Pass in tag or commit hash ver = 'HEAD' - git_call = ['git', '-C', path, 'ls-tree', '--full-tree', '-r', + git_call = ['git', '-C', path, 'ls-tree', '-r', '--name-only', ver] proc = run(git_call, stdout=PIPE, stderr=PIPE, text=True, timeout=5) if proc.returncode != 0: @@ -591,6 +594,9 @@ def populate_local_repo(path: str, parent=None, parent_source=None) -> list: return None for filepath in proc.stdout.splitlines(): + # unfetched submodules can list the contents of the higher level repository here. + if filepath.startswith('./') or filepath.startswith('..'): + continue if filepath in submodules: if parent is None: relative_path = filepath @@ -615,7 +621,7 @@ def populate_local_repo(path: str, parent=None, parent_source=None) -> list: return basedir.contents -def copy_remote_git_source(github_source: InstInfo, verbose: bool=True) -> SourceDir: +def copy_remote_git_source(github_source: InstInfo, verbose: bool=True, parent_source=None) -> SourceDir: """clone or fetch & checkout a local copy of a remote git repo""" user, repo = Source.get_github_user_repo(github_source.source_loc) if not user or not repo: @@ -635,7 +641,9 @@ def copy_remote_git_source(github_source: InstInfo, verbose: bool=True) -> Sourc assert _git_update(github_source.source_loc, local_path) else: _git_clone(github_source, local_path, verbose) - return SourceDir(local_path, srctype=Source.GIT_LOCAL_CLONE) + local_clone = SourceDir(local_path, srctype=Source.GIT_LOCAL_CLONE, parent_source=parent_source) + local_clone.populate() + return local_clone class Config(): @@ -2258,14 +2266,14 @@ def available_plugins() -> list: clone = copy_remote_git_source(InstInfo(None, source.original_source, source_dir=source.content), - verbose=False) + verbose=False, + parent_source=source) clone.srctype = Source.GIT_LOCAL_CLONE clone.parent_source = source if not clone: log.warning(f"could not clone github source {source.original_source}") continue source.local_clone = clone - source.local_clone.parent_source = source candidates.extend(find_plugin_candidates(source)) From 50ab5a8a02f55735111ad6d286df72412c73af6c Mon Sep 17 00:00:00 2001 From: Alex Myers Date: Thu, 15 Jan 2026 12:14:14 -0600 Subject: [PATCH 20/56] reckless: ensure repository submodules are always fetched Changelog-Fixed: Fixes an issue where reckless would misread the contents of an uncloned repository submodule. --- tools/reckless | 25 +++++++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/tools/reckless b/tools/reckless index 3f7f56c05f71..09e1ddabcf96 100755 --- a/tools/reckless +++ b/tools/reckless @@ -1248,6 +1248,14 @@ def _git_clone(src: InstInfo, dest: Union[PosixPath, str], verbose: bool=True) - remove_dir(str(dest)) log.error('Failed to clone repo') return False + + git = run(['git', 'submodule', 'update', '--init', '--recursive'], + cwd=str(dest), stdout=PIPE, stderr=PIPE, text=True, + check=False, timeout=120) + if git.returncode != 0: + log.warning(f'Failed to initialize submodules for {github_source}.') + return False + return True @@ -1278,7 +1286,6 @@ def _git_update(github_source: str, local_copy: PosixPath): git = run(['git', 'symbolic-ref', 'refs/remotes/origin/HEAD', '--short'], cwd=str(local_copy), stdout=PIPE, stderr=PIPE, text=True, check=False, timeout=60) - assert git.returncode == 0 if git.returncode != 0: return False default_branch = git.stdout.splitlines()[0] @@ -1290,8 +1297,16 @@ def _git_update(github_source: str, local_copy: PosixPath): git = run(['git', 'checkout', default_branch], cwd=str(local_copy), stdout=PIPE, stderr=PIPE, text=True, check=False, timeout=60) - assert git.returncode == 0 if git.returncode != 0: + log.warning(f'Failed to checkout branch {default_branch} of {github_source}.') + return False + + # Update all submodules to the referenced commit/branch/tag + git = run(['git', 'submodule', 'update', '--init', '--recursive'], + cwd=str(local_copy), stdout=PIPE, stderr=PIPE, text=True, + check=False, timeout=120) + if git.returncode != 0: + log.warning(f'Failed to initialize submodules for {github_source}.') return False return True @@ -2260,9 +2275,11 @@ def available_plugins() -> list: log.debug(f'confusing source: {source.type}') continue # It takes too many API calls to query for installable plugins accurately. - if source.type == Source.REMOTE_GIT_REPO and not source.local_clone: + if source.type == Source.REMOTE_GIT_REPO: # FIXME: ignoring non-cloned repos for now. - log.debug(f'cloning {source.original_source} in order to search') + if not source.local_clone: + log.debug(f'cloning {source.original_source} in order to search') + # Also updates existing clone and submodules clone = copy_remote_git_source(InstInfo(None, source.original_source, source_dir=source.content), From 80b01c4a987c7cd8326ea618bfa40d422afadd8a Mon Sep 17 00:00:00 2001 From: Alex Myers Date: Thu, 22 Jan 2026 15:21:59 -0600 Subject: [PATCH 21/56] reckless: Add logging port to transmit log messages in real time. This will be useful for lightning-rpc so that logs can be read while waiting on reckless to return json output once complete. --- tools/reckless | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/tools/reckless b/tools/reckless index 09e1ddabcf96..7dd7ff0d6d16 100755 --- a/tools/reckless +++ b/tools/reckless @@ -11,6 +11,7 @@ import logging import os from pathlib import Path, PosixPath import shutil +import socket from subprocess import Popen, PIPE, TimeoutExpired, run import tempfile import time @@ -51,11 +52,34 @@ class Logger: self.json_output = {"result": [], "log": []} self.capture = capture + self.socket = None + + def connect_socket(self, port: int): + """Streams log updates via this socket for lightningd notifications. + Used by the reckless-rpc plugin.""" + assert not self.socket + self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + try: + self.socket.connect(('localhost', port)) + except Exception as e: + logging.warning(f'socket failed to connect with {e}') + self.socket = None def str_esc(self, raw_string: str) -> str: assert isinstance(raw_string, str) return json.dumps(raw_string)[1:-1] + def push_to_socket(self, to_log: str, prefix: str): + if not self.socket or self.socket.fileno() <= 0: + return + try: + self.socket.sendall(f'{prefix}{to_log}\n'.encode('utf8')) + except Exception as e: + if self.capture: + self.json_output['log'].append(f'while feeding log to socket, encountered exception {e}') + else: + print(f'while feeding log to socket, encountered exception {e}') + def debug(self, to_log: str): assert isinstance(to_log, str) or hasattr(to_log, "__repr__") if logging.root.level > logging.DEBUG: @@ -65,6 +89,8 @@ class Logger: else: logging.debug(to_log) + self.push_to_socket(to_log, 'DEBUG: ') + def info(self, to_log: str): assert isinstance(to_log, str) or hasattr(to_log, "__repr__") if logging.root.level > logging.INFO: @@ -74,6 +100,8 @@ class Logger: else: print(to_log) + self.push_to_socket(to_log, 'INFO: ') + def warning(self, to_log: str): assert isinstance(to_log, str) or hasattr(to_log, "__repr__") if logging.root.level > logging.WARNING: @@ -83,6 +111,8 @@ class Logger: else: logging.warning(to_log) + self.push_to_socket(to_log, 'WARNING: ') + def error(self, to_log: str): assert isinstance(to_log, str) or hasattr(to_log, "__repr__") if logging.root.level > logging.ERROR: @@ -92,6 +122,8 @@ class Logger: else: logging.error(to_log) + self.push_to_socket(to_log, 'ERROR: ') + def add_result(self, result: Union[str, None]): assert json.dumps(result), "result must be json serializable" self.json_output["result"].append(result) @@ -2473,6 +2505,8 @@ if __name__ == '__main__': const=None) p.add_argument('-j', '--json', action=StoreTrueIdempotent, help='output in json format') + p.add_argument('--logging-port', action=StoreIdempotent, + help='lightning-rpc connects to this socket port to ingest log notifications') args = parser.parse_args() args = process_idempotent_args(args) @@ -2522,6 +2556,10 @@ if __name__ == '__main__': LIGHTNING_CONFIG = args.conf RECKLESS_CONFIG = load_config(reckless_dir=str(RECKLESS_DIR), network=NETWORK) + if args.logging_port: + log.connect_socket(int(args.logging_port)) + else: + log.debug('logging port argument not provided') GITHUB_COM = 'https://github.com' # Used for blackbox testing to avoid hitting github servers if 'REDIR_GITHUB' in os.environ: From 037f275c5992438e7686b699829585b67dada0cf Mon Sep 17 00:00:00 2001 From: Alex Myers Date: Wed, 28 Jan 2026 10:55:14 -0600 Subject: [PATCH 22/56] reckless-rpc: open socket for listening to streaming logs --- plugins/recklessrpc.c | 107 +++++++++++++++++++++++++++++++++++++++++- tools/reckless | 7 ++- 2 files changed, 111 insertions(+), 3 deletions(-) diff --git a/plugins/recklessrpc.c b/plugins/recklessrpc.c index 4815deb6d6d9..0a35cb2e3652 100644 --- a/plugins/recklessrpc.c +++ b/plugins/recklessrpc.c @@ -2,6 +2,7 @@ */ #include "config.h" +#include #include #include #include @@ -10,8 +11,10 @@ #include #include #include +#include #include #include +#include #include static struct plugin *plugin; @@ -21,12 +24,17 @@ struct reckless { int stdinfd; int stdoutfd; int stderrfd; + int logfd; char *stdoutbuf; char *stderrbuf; + char *logbuf; size_t stdout_read; /* running total */ size_t stdout_new; /* new since last read */ size_t stderr_read; size_t stderr_new; + size_t log_read; + size_t log_new; + char* log_to_process; pid_t pid; char *process_failed; }; @@ -51,7 +59,7 @@ static void reckless_send_yes(struct reckless *reckless) static struct io_plan *read_more(struct io_conn *conn, struct reckless *rkls) { rkls->stdout_read += rkls->stdout_new; - if (rkls->stdout_read == tal_count(rkls->stdoutbuf)) + if (rkls->stdout_read * 2 > tal_count(rkls->stdoutbuf)) tal_resize(&rkls->stdoutbuf, rkls->stdout_read * 2); return io_read_partial(conn, rkls->stdoutbuf + rkls->stdout_read, tal_count(rkls->stdoutbuf) - rkls->stdout_read, @@ -196,7 +204,7 @@ static struct io_plan *stderr_read_more(struct io_conn *conn, struct reckless *rkls) { rkls->stderr_read += rkls->stderr_new; - if (rkls->stderr_read == tal_count(rkls->stderrbuf)) + if (rkls->stderr_read * 2 > tal_count(rkls->stderrbuf)) tal_resize(&rkls->stderrbuf, rkls->stderr_read * 2); if (strends(rkls->stderrbuf, "[Y] to create one now.\n")) { plugin_log(plugin, LOG_DBG, "confirming config creation"); @@ -233,6 +241,82 @@ static bool is_single_arg_cmd(const char *command) { return false; } +static void log_conn_finish(struct io_conn *conn, struct reckless *reckless) +{ + io_close(conn); + close(reckless->logfd); + +} + +static struct io_plan *log_read_more(struct io_conn *conn, + struct reckless *rkls) +{ + rkls->log_read += rkls->log_new; + + if (rkls->log_read*2 >= tal_count(rkls->logbuf)) + tal_resize(&rkls->logbuf, rkls->log_read * 2); + + int unprocessed = rkls->log_read - (rkls->log_to_process - rkls->logbuf); + char *lineend = memchr(rkls->log_to_process, 0x0A, unprocessed); + + while (lineend != NULL) { + char * note; + note = tal_strndup(tmpctx, rkls->log_to_process, + lineend - rkls->log_to_process); + /* FIXME: Add notification for the utility logs. */ + plugin_log(plugin, LOG_DBG, "RECKLESS UTILITY: %s", note); + rkls->log_to_process = lineend + 1; + unprocessed = rkls->log_read - (rkls->log_to_process - rkls->logbuf); + lineend = memchr(rkls->log_to_process, 0x0A, unprocessed); + } + + return io_read_partial(conn, rkls->logbuf + rkls->log_read, + tal_count(rkls->logbuf) - rkls->log_read, + &rkls->log_new, log_read_more, rkls); +} + +static struct io_plan *log_conn_init(struct io_conn *conn, struct reckless *rkls) +{ + io_set_finish(conn, log_conn_finish, rkls); + return log_read_more(conn, rkls); +} + +static int open_socket(int *port) +{ + int sock; + sock = socket(AF_INET, SOCK_STREAM, 0); + if (sock < 0) { + plugin_log(plugin, LOG_UNUSUAL, "could not open socket for " + "streaming logs"); + return -1; + } + struct sockaddr_in ai; + ai.sin_family = AF_INET; + ai.sin_port = htons(0); + inet_pton(AF_INET, "127.0.0.1", &ai.sin_addr); + + if (bind(sock, (struct sockaddr *)&ai, sizeof(ai)) < 0) { + plugin_log(plugin, LOG_UNUSUAL, "failed to bind socket: %s", strerror(errno)); + close(sock); + return -1; + } + + socklen_t len = sizeof(ai); + if (getsockname(sock, (struct sockaddr *)&ai, &len) < 0) { + plugin_log(plugin, LOG_DBG, "couldn't retrieve socket port"); + return -1; + } + *port = ntohs(ai.sin_port); + + if (listen(sock, 64) != 0) { + plugin_log(plugin, LOG_UNUSUAL, "failed to listen on socket: %s", strerror(errno)); + close(sock); + return -1; + } + + return sock; +} + static struct command_result *reckless_call(struct command *cmd, const char *subcommand, const char *target, @@ -242,6 +326,13 @@ static struct command_result *reckless_call(struct command *cmd, if (!subcommand || !target) return command_fail(cmd, PLUGIN_ERROR, "invalid reckless call"); } + int sock; + int *port = tal(tmpctx, int); + sock = open_socket(port); + if (sock < 0) + plugin_log(plugin, LOG_BROKEN, "not streaming logs " + "from reckless utility"); + char **my_call; my_call = tal_arrz(tmpctx, char *, 0); tal_arr_expand(&my_call, "reckless"); @@ -251,6 +342,11 @@ static struct command_result *reckless_call(struct command *cmd, tal_arr_expand(&my_call, lconfig.lightningdir); tal_arr_expand(&my_call, "--network"); tal_arr_expand(&my_call, lconfig.network); + if (sock > 0) { + tal_arr_expand(&my_call, "--logging-port"); + tal_arr_expand(&my_call, tal_fmt(tmpctx, "%i", *port)); + } + if (lconfig.config) { tal_arr_expand(&my_call, "--conf"); tal_arr_expand(&my_call, lconfig.config); @@ -266,11 +362,17 @@ static struct command_result *reckless_call(struct command *cmd, reckless->cmd = cmd; reckless->stdoutbuf = tal_arrz(reckless, char, 4096); reckless->stderrbuf = tal_arrz(reckless, char, 4096); + reckless->logbuf = tal_arrz(reckless, char, 4096); reckless->stdout_read = 0; reckless->stdout_new = 0; reckless->stderr_read = 0; reckless->stderr_new = 0; + reckless->log_read = 0; + reckless->log_new = 0; + reckless->log_to_process = reckless->logbuf; reckless->process_failed = NULL; + reckless->logfd = sock; + char * full_cmd; full_cmd = tal_fmt(tmpctx, "calling:"); for (int i=0; istdoutfd, conn_init, reckless); io_new_conn(reckless, reckless->stderrfd, stderr_conn_init, reckless); + tal_free(my_call); return command_still_pending(cmd); } diff --git a/tools/reckless b/tools/reckless index 7dd7ff0d6d16..8cd714cf6454 100755 --- a/tools/reckless +++ b/tools/reckless @@ -62,8 +62,13 @@ class Logger: try: self.socket.connect(('localhost', port)) except Exception as e: - logging.warning(f'socket failed to connect with {e}') self.socket = None + if logging.root.level <= logging.WARNING: + msg = f'socket failed to connect with {e}' + if self.capture: + self.json_output['log'].append(self.str_esc(msg)) + else: + logging.warning(msg) def str_esc(self, raw_string: str) -> str: assert isinstance(raw_string, str) From 23f91ad1f53739c6d4f45c522e727eea77dabcb9 Mon Sep 17 00:00:00 2001 From: Alex Myers Date: Wed, 28 Jan 2026 15:40:47 -0600 Subject: [PATCH 23/56] reckless-rpc: publish reckless log notifications Changelog-added: The reckless-rpc plugin streams logs via the notification topic 'reckless_log' --- plugins/recklessrpc.c | 17 ++++++++++++++--- tests/plugins/custom_notifications.py | 5 +++++ tests/test_reckless.py | 17 +++++++++++++++-- 3 files changed, 34 insertions(+), 5 deletions(-) diff --git a/plugins/recklessrpc.c b/plugins/recklessrpc.c index 0a35cb2e3652..644ae6f725ff 100644 --- a/plugins/recklessrpc.c +++ b/plugins/recklessrpc.c @@ -241,6 +241,13 @@ static bool is_single_arg_cmd(const char *command) { return false; } +static void log_notify(char * log_line TAKES) +{ + struct json_stream *js = plugin_notification_start(NULL, "reckless_log"); + json_add_stringn(js, "log", log_line, tal_count(log_line)); + plugin_notification_end(plugin, js); +} + static void log_conn_finish(struct io_conn *conn, struct reckless *reckless) { io_close(conn); @@ -263,8 +270,8 @@ static struct io_plan *log_read_more(struct io_conn *conn, char * note; note = tal_strndup(tmpctx, rkls->log_to_process, lineend - rkls->log_to_process); - /* FIXME: Add notification for the utility logs. */ - plugin_log(plugin, LOG_DBG, "RECKLESS UTILITY: %s", note); + plugin_log(plugin, LOG_DBG, "reckless utility: %s", note); + log_notify(note); rkls->log_to_process = lineend + 1; unprocessed = rkls->log_read - (rkls->log_to_process - rkls->logbuf); lineend = memchr(rkls->log_to_process, 0x0A, unprocessed); @@ -445,6 +452,10 @@ static const struct plugin_command commands[] = { }, }; +static const char *notifications[] = { + "reckless_log", +}; + int main(int argc, char **argv) { setup_locale(); @@ -454,7 +465,7 @@ int main(int argc, char **argv) commands, ARRAY_SIZE(commands), NULL, 0, /* Notifications */ NULL, 0, /* Hooks */ - NULL, 0, /* Notification topics */ + notifications, ARRAY_SIZE(notifications), /* Notification topics */ NULL); /* plugin options */ return 0; diff --git a/tests/plugins/custom_notifications.py b/tests/plugins/custom_notifications.py index 1a3d92f18fc7..7ac27f763423 100755 --- a/tests/plugins/custom_notifications.py +++ b/tests/plugins/custom_notifications.py @@ -51,5 +51,10 @@ def on_faulty_emit(origin, payload, **kwargs): plugin.log("Got the ididntannouncethis event") +@plugin.subscribe("reckless_log") +def on_reckless_log(origin, **kwargs): + plugin.log("Got reckless_log: {}".format(kwargs)) + + plugin.add_notification_topic("custom") plugin.run() diff --git a/tests/test_reckless.py b/tests/test_reckless.py index 3bb1fde220f0..ce00e3f295d9 100644 --- a/tests/test_reckless.py +++ b/tests/test_reckless.py @@ -140,10 +140,10 @@ def reckless(cmds: list, dir: PosixPath = None, return RecklessResult(r, r.returncode, stdout, stderr) -def get_reckless_node(node_factory): +def get_reckless_node(node_factory, options={}, start=False): '''This may be unnecessary, but a preconfigured lightning dir is useful for reckless testing.''' - node = node_factory.get_node(options={}, start=False) + node = node_factory.get_node(options=options, start=start) return node @@ -461,3 +461,16 @@ def test_reckless_available(node_factory): assert r.search_stdout('testplugpass') assert r.search_stdout('testplugpyproj') assert r.search_stdout('testpluguv') + + +def test_reckless_notifications(node_factory): + """Reckless streams logs to the reckless-rpc plugin which are emitted + as 'reckless_log' notifications""" + notification_plugin = os.path.join(os.getcwd(), 'tests/plugins/custom_notifications.py') + node = get_reckless_node(node_factory, options={"plugin": notification_plugin}) + node.start() + listconfig_log = node.rpc.reckless('listconfig')['log'] + # Some trouble escaping the clone url for searching + listconfig_log.pop(1) + for log in listconfig_log: + assert node.daemon.is_in_log(f"reckless_log: {{'reckless_log': {{'log': '{log}'", start=0) From f6ecb7962f65d527859389b37fc74b0f2724e474 Mon Sep 17 00:00:00 2001 From: Alex Myers Date: Fri, 30 Jan 2026 16:55:44 -0600 Subject: [PATCH 24/56] reckless: close log socket before stdout Otherwise reckless-rpc can be concerned that the reckless utility process didn't exit cleanly. --- plugins/recklessrpc.c | 14 +++++++++++++- tests/test_reckless.py | 7 ++++++- tools/reckless | 3 +++ 3 files changed, 22 insertions(+), 2 deletions(-) diff --git a/plugins/recklessrpc.c b/plugins/recklessrpc.c index 644ae6f725ff..2f9945d94e9d 100644 --- a/plugins/recklessrpc.c +++ b/plugins/recklessrpc.c @@ -69,6 +69,7 @@ static struct io_plan *read_more(struct io_conn *conn, struct reckless *rkls) static struct command_result *reckless_result(struct io_conn *conn, struct reckless *reckless) { + io_close(conn); struct json_stream *response; if (reckless->process_failed) { response = jsonrpc_stream_fail(reckless->cmd, @@ -152,6 +153,16 @@ static void reckless_conn_finish(struct io_conn *conn, /* FIXME: avoid EBADFD - leave stdin fd open? */ if (errno && errno != 9) plugin_log(plugin, LOG_DBG, "err: %s", strerror(errno)); + struct pollfd pfd = { .fd = reckless->logfd, .events = POLLIN }; + poll(&pfd, 1, 20); // wait for any remaining log data + + /* Close the log streaming socket. */ + if (reckless->logfd) { + if (close(reckless->logfd) != 0) + plugin_log(plugin, LOG_DBG, "closing log socket failed: %s", strerror(errno)); + reckless->logfd = 0; + } + if (reckless->pid > 0) { int status = 0; pid_t p; @@ -160,6 +171,7 @@ static void reckless_conn_finish(struct io_conn *conn, if (p != reckless->pid && reckless->pid) { plugin_log(plugin, LOG_DBG, "reckless failed to exit, " "killing now."); + io_close(conn); kill(reckless->pid, SIGKILL); reckless_fail(reckless, "reckless process hung"); /* Reckless process exited and with normal status? */ @@ -251,7 +263,7 @@ static void log_notify(char * log_line TAKES) static void log_conn_finish(struct io_conn *conn, struct reckless *reckless) { io_close(conn); - close(reckless->logfd); + reckless->logfd = 0; } diff --git a/tests/test_reckless.py b/tests/test_reckless.py index ce00e3f295d9..bff510e6f418 100644 --- a/tests/test_reckless.py +++ b/tests/test_reckless.py @@ -419,7 +419,7 @@ def test_tag_install(node_factory): # Note: uv timeouts from the GH network seem to happen? @pytest.mark.slow_test -@pytest.mark.flaky(reruns=3) +@pytest.mark.flaky(max_runs=3) def test_reckless_uv_install(node_factory): node = get_reckless_node(node_factory) node.start() @@ -468,6 +468,11 @@ def test_reckless_notifications(node_factory): as 'reckless_log' notifications""" notification_plugin = os.path.join(os.getcwd(), 'tests/plugins/custom_notifications.py') node = get_reckless_node(node_factory, options={"plugin": notification_plugin}) + NETWORK = os.environ.get('TEST_NETWORK') + if not NETWORK: + NETWORK = 'regtest' + reckless(['listconfig', f'--network={NETWORK}', '--json'], + dir=node.lightning_dir) node.start() listconfig_log = node.rpc.reckless('listconfig')['log'] # Some trouble escaping the clone url for searching diff --git a/tools/reckless b/tools/reckless index 8cd714cf6454..933498131c02 100755 --- a/tools/reckless +++ b/tools/reckless @@ -2597,3 +2597,6 @@ if __name__ == '__main__': if log.capture: log.reply_json() + # We're done streaming to this socket, but the rpc plugin will close it. + if log.socket: + log.socket.shutdown(socket.SHUT_WR) From 29a84e0422df7f9ee23f9149fe27fc7c231bcb21 Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Fri, 6 Feb 2026 10:14:26 +1030 Subject: [PATCH 25/56] plugins/recklessrpc: use membuf for handling socket input. This is the standard battle-tested way of doing it, and it avoids the various bugs in the open-coded implementation. Inspired by common/jsonrpc_io.c which does a similar thing for JSON. Signed-off-by: Rusty Russell --- plugins/recklessrpc.c | 75 +++++++++++++++++++++++++------------------ 1 file changed, 44 insertions(+), 31 deletions(-) diff --git a/plugins/recklessrpc.c b/plugins/recklessrpc.c index 2f9945d94e9d..492e15603618 100644 --- a/plugins/recklessrpc.c +++ b/plugins/recklessrpc.c @@ -5,6 +5,7 @@ #include #include #include +#include #include #include #include @@ -27,16 +28,16 @@ struct reckless { int logfd; char *stdoutbuf; char *stderrbuf; - char *logbuf; size_t stdout_read; /* running total */ size_t stdout_new; /* new since last read */ size_t stderr_read; size_t stderr_new; - size_t log_read; - size_t log_new; - char* log_to_process; pid_t pid; char *process_failed; + + MEMBUF(char) logbuf; + /* Amount just read by io_read_partial */ + size_t logbytes_read; }; struct lconfig { @@ -253,10 +254,10 @@ static bool is_single_arg_cmd(const char *command) { return false; } -static void log_notify(char * log_line TAKES) +static void log_notify(const char *log_line, size_t len) { struct json_stream *js = plugin_notification_start(NULL, "reckless_log"); - json_add_stringn(js, "log", log_line, tal_count(log_line)); + json_add_stringn(js, "log", log_line, len); plugin_notification_end(plugin, js); } @@ -267,31 +268,43 @@ static void log_conn_finish(struct io_conn *conn, struct reckless *reckless) } +/* len does NOT include the \n */ +static const char *get_line(const struct reckless *rkls, size_t *len) +{ + const char *line = membuf_elems(&rkls->logbuf); + const char *eol = memchr(line, '\n', membuf_num_elems(&rkls->logbuf)); + + if (eol) { + *len = eol - line; + return line; + } + return NULL; +} + static struct io_plan *log_read_more(struct io_conn *conn, - struct reckless *rkls) + struct reckless *rkls) { - rkls->log_read += rkls->log_new; - - if (rkls->log_read*2 >= tal_count(rkls->logbuf)) - tal_resize(&rkls->logbuf, rkls->log_read * 2); - - int unprocessed = rkls->log_read - (rkls->log_to_process - rkls->logbuf); - char *lineend = memchr(rkls->log_to_process, 0x0A, unprocessed); - - while (lineend != NULL) { - char * note; - note = tal_strndup(tmpctx, rkls->log_to_process, - lineend - rkls->log_to_process); - plugin_log(plugin, LOG_DBG, "reckless utility: %s", note); - log_notify(note); - rkls->log_to_process = lineend + 1; - unprocessed = rkls->log_read - (rkls->log_to_process - rkls->logbuf); - lineend = memchr(rkls->log_to_process, 0x0A, unprocessed); + size_t len; + const char *line; + + /* We read some more stuff in! */ + membuf_added(&rkls->logbuf, rkls->logbytes_read); + rkls->logbytes_read = 0; + + while ((line = get_line(rkls, &len)) != NULL) { + plugin_log(plugin, LOG_DBG, "reckless utility: %.*s", (int)len, line); + log_notify(line, len); + membuf_consume(&rkls->logbuf, len + 1); } - return io_read_partial(conn, rkls->logbuf + rkls->log_read, - tal_count(rkls->logbuf) - rkls->log_read, - &rkls->log_new, log_read_more, rkls); + /* Make sure there's more room */ + membuf_prepare_space(&rkls->logbuf, 4096); + + return io_read_partial(conn, + membuf_space(&rkls->logbuf), + membuf_num_space(&rkls->logbuf), + &rkls->logbytes_read, + log_read_more, rkls); } static struct io_plan *log_conn_init(struct io_conn *conn, struct reckless *rkls) @@ -381,16 +394,16 @@ static struct command_result *reckless_call(struct command *cmd, reckless->cmd = cmd; reckless->stdoutbuf = tal_arrz(reckless, char, 4096); reckless->stderrbuf = tal_arrz(reckless, char, 4096); - reckless->logbuf = tal_arrz(reckless, char, 4096); reckless->stdout_read = 0; reckless->stdout_new = 0; reckless->stderr_read = 0; reckless->stderr_new = 0; - reckless->log_read = 0; - reckless->log_new = 0; - reckless->log_to_process = reckless->logbuf; reckless->process_failed = NULL; reckless->logfd = sock; + membuf_init(&reckless->logbuf, + tal_arr(reckless, char, 10), + 10, membuf_tal_resize); + reckless->logbytes_read = 0; char * full_cmd; full_cmd = tal_fmt(tmpctx, "calling:"); From 1c3f6b2b72f1e0b81c486f3289e460c9a6c7fe9e Mon Sep 17 00:00:00 2001 From: Alex Myers Date: Wed, 11 Feb 2026 09:25:19 -0600 Subject: [PATCH 26/56] reckless: json escape streaming logs --- tools/reckless | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/tools/reckless b/tools/reckless index 933498131c02..c872c4c1ff5a 100755 --- a/tools/reckless +++ b/tools/reckless @@ -71,7 +71,9 @@ class Logger: logging.warning(msg) def str_esc(self, raw_string: str) -> str: - assert isinstance(raw_string, str) + assert isinstance(raw_string, str) or hasattr(to_log, "__repr__") + if not isinstance(raw_string, str): + return json.dumps(str(raw_string))[1:-1] return json.dumps(raw_string)[1:-1] def push_to_socket(self, to_log: str, prefix: str): @@ -94,7 +96,7 @@ class Logger: else: logging.debug(to_log) - self.push_to_socket(to_log, 'DEBUG: ') + self.push_to_socket(self.str_esc(to_log), 'DEBUG: ') def info(self, to_log: str): assert isinstance(to_log, str) or hasattr(to_log, "__repr__") @@ -105,7 +107,7 @@ class Logger: else: print(to_log) - self.push_to_socket(to_log, 'INFO: ') + self.push_to_socket(self.str_esc(to_log), 'INFO: ') def warning(self, to_log: str): assert isinstance(to_log, str) or hasattr(to_log, "__repr__") @@ -116,7 +118,7 @@ class Logger: else: logging.warning(to_log) - self.push_to_socket(to_log, 'WARNING: ') + self.push_to_socket(self.str_esc(to_log), 'WARNING: ') def error(self, to_log: str): assert isinstance(to_log, str) or hasattr(to_log, "__repr__") @@ -127,7 +129,7 @@ class Logger: else: logging.error(to_log) - self.push_to_socket(to_log, 'ERROR: ') + self.push_to_socket(self.str_esc(to_log), 'ERROR: ') def add_result(self, result: Union[str, None]): assert json.dumps(result), "result must be json serializable" @@ -1531,7 +1533,7 @@ def _install_plugin(src: InstInfo) -> Union[InstInfo, None]: # Create symlink in staging tree to redirect to the plugins entrypoint log.debug(f"linking source {staging_path / cloned_src.entry} to " f"{Path(staged_src.source_loc) / cloned_src.entry}") - log.debug(staged_src) + log.debug(str(staged_src)) (Path(staged_src.source_loc) / cloned_src.entry).\ symlink_to(staging_path / cloned_src.entry) @@ -1884,7 +1886,7 @@ def enable(plugin_name: str): return None else: log.error(f'reckless: {inst.name} failed to start!') - log.error(err) + log.error(str(err)) return None except RPCError: log.info(('lightningd rpc unavailable. ' From 93f7e8221fa923d3ae88723b3f3baf6deab80894 Mon Sep 17 00:00:00 2001 From: Alex Myers Date: Wed, 11 Feb 2026 09:27:23 -0600 Subject: [PATCH 27/56] reklessrpc: duplicate json format of reckless listavailable output --- plugins/recklessrpc.c | 62 ++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 58 insertions(+), 4 deletions(-) diff --git a/plugins/recklessrpc.c b/plugins/recklessrpc.c index 492e15603618..89e86edf194f 100644 --- a/plugins/recklessrpc.c +++ b/plugins/recklessrpc.c @@ -67,6 +67,51 @@ static struct io_plan *read_more(struct io_conn *conn, struct reckless *rkls) &rkls->stdout_new, read_more, rkls); } +static void dup_listavailable_result(struct reckless *reckless, + struct json_stream *response, + char *reckless_result, + const jsmntok_t *results_tok) +{ + json_array_start(response, "result"); + size_t plugins, requirements; + const jsmntok_t *result, *requirement, *requirements_tok; + const char *plugin_name, *short_description, *long_description, *entrypoint; + + json_for_each_arr(plugins, result, results_tok) { + json_object_start(response, NULL); + + json_scan(tmpctx, reckless_result, result, + "{name:%," + "short_description:%," + "long_description:%," + "entrypoint:%}", + JSON_SCAN_TAL(tmpctx, json_strdup, &plugin_name), + JSON_SCAN_TAL(tmpctx, json_strdup, &short_description), + JSON_SCAN_TAL(tmpctx, json_strdup, &long_description), + JSON_SCAN_TAL(tmpctx, json_strdup, &entrypoint)); + + json_add_string(response, "name", plugin_name); + if (!streq(short_description, "null")) + json_add_string(response, "short_description", short_description); + if (!streq(long_description, "null")) + json_add_string(response, "long_description", long_description); + json_add_string(response, "entypoint", entrypoint); + + json_array_start(response, "requirements"); + requirements_tok = json_get_member(reckless_result, result, "requirements"); + if (requirements_tok) { + json_for_each_arr(requirements, requirement, requirements_tok) { + json_add_string(response, NULL, + json_strdup(tmpctx, reckless_result, requirement)); + } + } + json_array_end(response); + + json_object_end(response); + } + json_array_end(response); +} + static struct command_result *reckless_result(struct io_conn *conn, struct reckless *reckless) { @@ -78,7 +123,10 @@ static struct command_result *reckless_result(struct io_conn *conn, reckless->process_failed); return command_finished(reckless->cmd, response); } - const jsmntok_t *results, *result, *logs, *log, *conf; + + /* The reckless utility outputs utf-8 and ends the transmission with + * \u0004, which jsmn is unable to parse. */ + const jsmntok_t *results, *result, *logs, *log, *conf, *next; size_t i; jsmn_parser parser; jsmntok_t *toks; @@ -90,7 +138,7 @@ static struct command_result *reckless_result(struct io_conn *conn, const char *err; if (res == JSMN_ERROR_INVAL) err = tal_fmt(tmpctx, "reckless returned invalid character in json " - "output"); + "output. (total length %lu)", strlen(reckless->stdoutbuf)); else if (res == JSMN_ERROR_PART) err = tal_fmt(tmpctx, "reckless returned partial output"); else if (res == JSMN_ERROR_NOMEM ) @@ -100,7 +148,8 @@ static struct command_result *reckless_result(struct io_conn *conn, err = NULL; if (err) { - plugin_log(plugin, LOG_UNUSUAL, "failed to parse json: %s", err); + if (res == JSMN_ERROR_INVAL) + plugin_log(plugin, LOG_BROKEN, "invalid char in json"); response = jsonrpc_stream_fail(reckless->cmd, PLUGIN_ERROR, err); return command_finished(reckless->cmd, response); @@ -108,15 +157,20 @@ static struct command_result *reckless_result(struct io_conn *conn, response = jsonrpc_stream_success(reckless->cmd); results = json_get_member(reckless->stdoutbuf, toks, "result"); + next = json_get_arr(results, 0); conf = json_get_member(reckless->stdoutbuf, results, "requested_lightning_conf"); if (conf) { - plugin_log(plugin, LOG_DBG, "dealing with listconfigs output"); + plugin_log(plugin, LOG_DBG, "ingesting listconfigs output"); json_object_start(response, "result"); json_for_each_obj(i, result, results) { json_add_tok(response, json_strdup(tmpctx, reckless->stdoutbuf, result), result+1, reckless->stdoutbuf); } json_object_end(response); + } else if (next && next->type == JSMN_OBJECT) { + plugin_log(plugin, LOG_DBG, "ingesting listavailable output"); + dup_listavailable_result(reckless, response, reckless->stdoutbuf, results); + } else { json_array_start(response, "result"); json_for_each_arr(i, result, results) { From 290f8415167116dbaeca01c38c10106f62f4ddbe Mon Sep 17 00:00:00 2001 From: Alex Myers Date: Thu, 12 Feb 2026 11:27:05 -0600 Subject: [PATCH 28/56] recklessrpc: fixup close connection --- plugins/recklessrpc.c | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/plugins/recklessrpc.c b/plugins/recklessrpc.c index 89e86edf194f..455822865a87 100644 --- a/plugins/recklessrpc.c +++ b/plugins/recklessrpc.c @@ -112,10 +112,8 @@ static void dup_listavailable_result(struct reckless *reckless, json_array_end(response); } -static struct command_result *reckless_result(struct io_conn *conn, - struct reckless *reckless) +static struct command_result *reckless_result(struct reckless *reckless) { - io_close(conn); struct json_stream *response; if (reckless->process_failed) { response = jsonrpc_stream_fail(reckless->cmd, @@ -205,6 +203,7 @@ static struct command_result *reckless_fail(struct reckless *reckless, static void reckless_conn_finish(struct io_conn *conn, struct reckless *reckless) { + io_close(conn); /* FIXME: avoid EBADFD - leave stdin fd open? */ if (errno && errno != 9) plugin_log(plugin, LOG_DBG, "err: %s", strerror(errno)); @@ -226,15 +225,13 @@ static void reckless_conn_finish(struct io_conn *conn, if (p != reckless->pid && reckless->pid) { plugin_log(plugin, LOG_DBG, "reckless failed to exit, " "killing now."); - io_close(conn); kill(reckless->pid, SIGKILL); reckless_fail(reckless, "reckless process hung"); /* Reckless process exited and with normal status? */ } else if (WIFEXITED(status) && !WEXITSTATUS(status)) { plugin_log(plugin, LOG_DBG, - "Reckless subprocess complete: %s", - reckless->stdoutbuf); - reckless_result(conn, reckless); + "Reckless subprocess complete"); + reckless_result(reckless); /* Don't try to process json if python raised an error. */ } else { plugin_log(plugin, LOG_DBG, "%s", reckless->stderrbuf); @@ -252,7 +249,6 @@ static void reckless_conn_finish(struct io_conn *conn, "The reckless subprocess has failed."); } } - io_close(conn); tal_free(reckless); } From f3468abffaeacbd507bba7c486bd8f45963ffdc7 Mon Sep 17 00:00:00 2001 From: Alex Myers Date: Wed, 11 Feb 2026 16:14:06 -0600 Subject: [PATCH 29/56] recklessrpc: fix buffer resize --- plugins/recklessrpc.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/recklessrpc.c b/plugins/recklessrpc.c index 455822865a87..2ce151c0c902 100644 --- a/plugins/recklessrpc.c +++ b/plugins/recklessrpc.c @@ -60,8 +60,8 @@ static void reckless_send_yes(struct reckless *reckless) static struct io_plan *read_more(struct io_conn *conn, struct reckless *rkls) { rkls->stdout_read += rkls->stdout_new; - if (rkls->stdout_read * 2 > tal_count(rkls->stdoutbuf)) - tal_resize(&rkls->stdoutbuf, rkls->stdout_read * 2); + while (rkls->stdout_read >= tal_count(rkls->stdoutbuf)) + tal_resizez(&rkls->stdoutbuf, tal_count(rkls->stdoutbuf) * 2); return io_read_partial(conn, rkls->stdoutbuf + rkls->stdout_read, tal_count(rkls->stdoutbuf) - rkls->stdout_read, &rkls->stdout_new, read_more, rkls); From a6c0dc9f9c7ce7aece0b8567b2cc26d7e7537af2 Mon Sep 17 00:00:00 2001 From: Alex Myers Date: Thu, 12 Feb 2026 11:41:04 -0600 Subject: [PATCH 30/56] pytest: skip reckless uv install under valgrind --- tests/test_reckless.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/test_reckless.py b/tests/test_reckless.py index bff510e6f418..099c715dbc41 100644 --- a/tests/test_reckless.py +++ b/tests/test_reckless.py @@ -292,6 +292,7 @@ def test_install(node_factory): assert r.returncode == 0 +@unittest.skipIf(VALGRIND, "virtual environment triggers memleak detection") def test_install_cleanup(node_factory): """test failed installation and post install cleanup""" n = get_reckless_node(node_factory) @@ -419,6 +420,7 @@ def test_tag_install(node_factory): # Note: uv timeouts from the GH network seem to happen? @pytest.mark.slow_test +@unittest.skipIf(VALGRIND, "node too slow for starting plugin under valgrind") @pytest.mark.flaky(max_runs=3) def test_reckless_uv_install(node_factory): node = get_reckless_node(node_factory) From e140e2ff93eea4a7966f99b796d89430cd48ae86 Mon Sep 17 00:00:00 2001 From: Alex Myers Date: Fri, 12 Dec 2025 10:47:54 -0600 Subject: [PATCH 31/56] reckless: don't be confused by python deps in rust plugins Python dependencies are often used for the test framework. Checking other installers first means they're less likely to be misinterpretted as python plugins. --- tools/reckless | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tools/reckless b/tools/reckless index c872c4c1ff5a..78412f4d4877 100755 --- a/tools/reckless +++ b/tools/reckless @@ -1204,8 +1204,10 @@ shebang.add_entrypoint('{name}') # An extra installable check to see if a #! is present in the file shebang.check = check_for_shebang -INSTALLERS = [shebang, pythonuv, pythonuvlegacy, python3venv, poetryvenv, - pyprojectViaPip, nodejs, rust_cargo] +# Projects may include python dependencies for testing, so give other installers +# first priority. +INSTALLERS = [rust_cargo, nodejs, shebang, pythonuv, pythonuvlegacy, python3venv, + poetryvenv, pyprojectViaPip] def help_alias(targets: list): From 7fbbdb02fd2f8859ba809c685e8eda06b95b9882 Mon Sep 17 00:00:00 2001 From: Alex Myers Date: Mon, 15 Dec 2025 17:26:29 -0600 Subject: [PATCH 32/56] reckless: catch and raise usage errors While the reckless utility provided usage hints, the rpc plugin would simply exit with an unhelpful: { "code": -3, "message": "the reckless process has crashed" } This captures the usage hint from the utility and reports it from the plugin as well. Changelog-Fixed: reckless-rpc plugin now raises incorrect usage from the reckless utility. --- plugins/recklessrpc.c | 37 +++++++++++++++++++++++++++++-------- tests/test_reckless.py | 19 +++++++++++++++++++ 2 files changed, 48 insertions(+), 8 deletions(-) diff --git a/plugins/recklessrpc.c b/plugins/recklessrpc.c index 2ce151c0c902..c60bbfe12374 100644 --- a/plugins/recklessrpc.c +++ b/plugins/recklessrpc.c @@ -200,6 +200,23 @@ static struct command_result *reckless_fail(struct reckless *reckless, return command_finished(reckless->cmd, resp); } +/* Regurgitates the syntax error reported by the utility */ +static struct command_result *fail_bad_usage(struct reckless *reckless) +{ + char **lines; + lines = tal_strsplit(reckless, reckless->stderrbuf, "\n", STR_EMPTY_OK); + if (lines != NULL) + { + /* The last line of reckless output contains the usage error. + * Capture it for the user. */ + int i = 0; + while (lines[i + 1] != NULL) + i++; + return reckless_fail(reckless, lines[i]); + } + return reckless_fail(reckless, "the reckless process has crashed"); +} + static void reckless_conn_finish(struct io_conn *conn, struct reckless *reckless) { @@ -239,14 +256,18 @@ static void reckless_conn_finish(struct io_conn *conn, "Reckless process has crashed (%i).", WEXITSTATUS(status)); char * err; - if (reckless->process_failed) - err = reckless->process_failed; - else - err = tal_strdup(tmpctx, "the reckless process " - "has crashed"); - reckless_fail(reckless, err); - plugin_log(plugin, LOG_UNUSUAL, - "The reckless subprocess has failed."); + if (WEXITSTATUS(status) == 2) + fail_bad_usage(reckless); + else { + if (reckless->process_failed) + err = reckless->process_failed; + else + err = tal_strdup(tmpctx, "the reckless process " + "has crashed"); + reckless_fail(reckless, err); + plugin_log(plugin, LOG_UNUSUAL, + "The reckless subprocess has failed."); + } } } tal_free(reckless); diff --git a/tests/test_reckless.py b/tests/test_reckless.py index 099c715dbc41..0fc9d2fa35c1 100644 --- a/tests/test_reckless.py +++ b/tests/test_reckless.py @@ -6,6 +6,7 @@ import time import unittest from fixtures import * # noqa: F401,F403 +from pyln.client import lightning from pyln.testing.utils import VALGRIND import pytest @@ -481,3 +482,21 @@ def test_reckless_notifications(node_factory): listconfig_log.pop(1) for log in listconfig_log: assert node.daemon.is_in_log(f"reckless_log: {{'reckless_log': {{'log': '{log}'", start=0) + + +def test_reckless_usage(node_factory): + """The reckless rpc response is more useful if it can pass back incorrect + usage errors.""" + node = node_factory.get_node(options={}, may_fail=True, start=False) + node.start(stderr_redir=True) + r = reckless(['searhc', 'testplugpass'], + dir=node.lightning_dir) + # The reckless utility should fail and argparse should provide a usage hint + # as the line of output. + assert r.returncode == 2 + assert "reckless: error: argument cmd1: invalid choice: 'searhc' (choose from " in r.stderr[-1] + + # The rpc plugin should capture and raise this usage error + with pytest.raises(lightning.RpcError, + match="reckless: error: argument cmd1: invalid choice: 'saerch'"): + node.rpc.reckless('saerch', 'testplugpass') From f8d08681055dfad30e3b361256929b41eef9f416 Mon Sep 17 00:00:00 2001 From: Alex Myers Date: Thu, 12 Feb 2026 14:52:33 -0600 Subject: [PATCH 33/56] pytest: add timeout to reckless tests Trying to debug pytest teardown under the github CI runner. --- tests/test_reckless.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/test_reckless.py b/tests/test_reckless.py index 0fc9d2fa35c1..bc5b0c765b2a 100644 --- a/tests/test_reckless.py +++ b/tests/test_reckless.py @@ -157,6 +157,7 @@ def test_basic_help(): assert r.search_stdout("options:") or r.search_stdout("optional arguments:") +@pytest.mark.timeout(120) def test_reckless_version_listconfig(node_factory): '''Version should be reported without loading config and should advance with lightningd.''' @@ -466,6 +467,7 @@ def test_reckless_available(node_factory): assert r.search_stdout('testpluguv') +@pytest.mark.timeout(120) def test_reckless_notifications(node_factory): """Reckless streams logs to the reckless-rpc plugin which are emitted as 'reckless_log' notifications""" From b50601200b3ffb596f3429f00d0a2e4e96c53d7e Mon Sep 17 00:00:00 2001 From: Alex Myers Date: Thu, 12 Feb 2026 16:22:14 -0600 Subject: [PATCH 34/56] reckless: update json schema with listavailable, listconfig, listinstalled commands --- contrib/msggen/msggen/schema.json | 3 +++ doc/reckless.7.md | 2 +- doc/schemas/reckless.json | 3 +++ 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/contrib/msggen/msggen/schema.json b/contrib/msggen/msggen/schema.json index 093a98666d12..11cdec0983d1 100644 --- a/contrib/msggen/msggen/schema.json +++ b/contrib/msggen/msggen/schema.json @@ -30503,6 +30503,9 @@ "enable", "disable", "source", + "listavailable", + "listconfig", + "listinstalled", "--version" ], "description": [ diff --git a/doc/reckless.7.md b/doc/reckless.7.md index acf5f4cd40d9..39f231ae47e2 100644 --- a/doc/reckless.7.md +++ b/doc/reckless.7.md @@ -11,7 +11,7 @@ DESCRIPTION The **reckless** RPC starts a reckless process with the *command* and *target* provided. Node configuration, network, and lightning direrctory are automatically passed to the reckless utility. -- **command** (string) (one of "install", "uninstall", "search", "enable", "disable", "source", "--version"): Determines which command to pass to reckless +- **command** (string) (one of "install", "uninstall", "search", "enable", "disable", "source", "listavailable", "listconfig", "listinstalled", "--version"): Determines which command to pass to reckless - *command* **install** takes a *plugin\_name* to search for and install a named plugin. - *command* **uninstall** takes a *plugin\_name* and attempts to uninstall a plugin of the same name. - *command* **search** takes a *plugin\_name* to search for a named plugin. diff --git a/doc/schemas/reckless.json b/doc/schemas/reckless.json index 372eb772dd3e..27bf2f8f5419 100644 --- a/doc/schemas/reckless.json +++ b/doc/schemas/reckless.json @@ -21,6 +21,9 @@ "enable", "disable", "source", + "listavailable", + "listconfig", + "listinstalled", "--version" ], "description": [ From 70d42078ccbe5105e235b4b8be8b6b017505cc65 Mon Sep 17 00:00:00 2001 From: Alex Myers Date: Fri, 13 Feb 2026 10:30:05 -0600 Subject: [PATCH 35/56] pytest: make sure tools/reckless is available on PATH so that reckless-rpc will use the correct executable. --- tests/test_reckless.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/tests/test_reckless.py b/tests/test_reckless.py index bc5b0c765b2a..71ef7a2a2be6 100644 --- a/tests/test_reckless.py +++ b/tests/test_reckless.py @@ -2,6 +2,7 @@ import os from pathlib import PosixPath, Path import re +import shutil import subprocess import time import unittest @@ -79,6 +80,16 @@ def canned_github_server(directory): f.write(f"pyln-client\n\n") +@pytest.fixture(autouse=True) +def add_reckless_to_env_path(): + """Allows the reckless-rpc plugin to use the reckless executable from the + build directory, rather than whatever it finds already installed.""" + current_path = os.environ['PATH'] + tools_dir = Path(os.path.dirname(os.path.realpath(__file__))).parent / 'tools' + os.environ['PATH'] = f'{tools_dir}:{current_path}' + assert shutil.which('reckless') + + class RecklessResult: def __init__(self, process, returncode, stdout, stderr): self.process = process From fa9168c29afec8610c8a6bc4c35c3e1eaa64a6fc Mon Sep 17 00:00:00 2001 From: enaples Date: Wed, 18 Feb 2026 10:22:29 +0100 Subject: [PATCH 36/56] rebased --- tools/reckless | 99 ++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 91 insertions(+), 8 deletions(-) diff --git a/tools/reckless b/tools/reckless index 78412f4d4877..5182e7898f48 100755 --- a/tools/reckless +++ b/tools/reckless @@ -385,9 +385,8 @@ class SourceDir(): self.contents = populate_local_dir(self.location) elif self.srctype in [Source.LOCAL_REPO, Source.GIT_LOCAL_CLONE]: self.contents = populate_local_repo(self.location, parent=self, parent_source=self.parent_source) - elif self.srctype == Source.REMOTE_GIT_REPO: - self.contents = copy_remote_git_source(InstInfo(self.name, self.location)).contents - + elif self.srctype == Source.GITHUB_REPO: + self.contents = populate_github_repo(self.location) else: raise Exception("populate method undefined for {self.srctype}") # Ensure the relative path of the contents is inherited. @@ -660,7 +659,92 @@ def populate_local_repo(path: str, parent=None, parent_source=None) -> list: return basedir.contents -def copy_remote_git_source(github_source: InstInfo, verbose: bool=True, parent_source=None) -> SourceDir: +def source_element_from_repo_api(member: dict): + # api accessed via /contents/ + if 'type' in member and 'name' in member and 'git_url' in member: + if member['type'] == 'dir': + return SourceDir(member['git_url'], srctype=Source.GITHUB_REPO, + name=member['name']) + elif member['type'] == 'file': + # Likely a submodule + if member['size'] == 0: + return SourceDir(None, srctype=Source.GITHUB_REPO, + name=member['name']) + return SourceFile(member['name']) + elif member['type'] == 'commit': + # No path is given by the api here + return SourceDir(None, srctype=Source.GITHUB_REPO, + name=member['name']) + # git_url with /tree/ presents results a little differently + elif 'type' in member and 'path' in member and 'url' in member: + if member['type'] not in ['tree', 'blob']: + log.debug(f' skipping {member["path"]} type={member["type"]}') + if member['type'] == 'tree': + return SourceDir(member['url'], srctype=Source.GITHUB_REPO, + name=member['path']) + elif member['type'] == 'blob': + # This can be a submodule + if member['size'] == 0: + return SourceDir(member['git_url'], srctype=Source.GITHUB_REPO, + name=member['name']) + return SourceFile(member['path']) + elif member['type'] == 'commit': + # No path is given by the api here + return SourceDir(None, srctype=Source.GITHUB_REPO, + name=member['name']) + return None + + +def populate_github_repo(url: str) -> list: + """populate one level of a github repository via REST API""" + # Forces search to clone remote repos (for blackbox testing) + if GITHUB_API_FALLBACK: + with tempfile.NamedTemporaryFile() as tmp: + raise HTTPError(url, 403, 'simulated ratelimit', {}, tmp) + # FIXME: This probably contains leftover cruft. + repo = url.split('/') + while '' in repo: + repo.remove('') + repo_name = None + parsed_url = urlparse(url.removesuffix('.git')) + if 'github.com' not in parsed_url.netloc: + return None + if len(parsed_url.path.split('/')) < 2: + return None + start = 1 + # Maybe we were passed an api.github.com/repo/ url + if 'api' in parsed_url.netloc: + start += 1 + repo_user = parsed_url.path.split('/')[start] + repo_name = parsed_url.path.split('/')[start + 1] + + # Get details from the github API. + if API_GITHUB_COM in url: + api_url = url + else: + api_url = f'{API_GITHUB_COM}/repos/{repo_user}/{repo_name}/contents/' + + git_url = api_url + if "api.github.com" in git_url: + # This lets us redirect to handle blackbox testing + log.debug(f'fetching from gh API: {git_url}') + git_url = (API_GITHUB_COM + git_url.split("api.github.com")[-1]) + # Ratelimiting occurs for non-authenticated GH API calls at 60 in 1 hour. + r = urlopen(git_url, timeout=5) + if r.status != 200: + return False + if 'git/tree' in git_url: + tree = json.loads(r.read().decode())['tree'] + else: + tree = json.loads(r.read().decode()) + contents = [] + for sub in tree: + if source_element_from_repo_api(sub): + contents.append(source_element_from_repo_api(sub)) + return contents + + +def copy_remote_git_source(github_source: InstInfo, verbose: bool=True) -> SourceDir: """clone or fetch & checkout a local copy of a remote git repo""" user, repo = Source.get_github_user_repo(github_source.source_loc) if not user or not repo: @@ -2252,7 +2336,7 @@ def fetch_manifest(source: SourceDir) -> dict: return None with open(path, 'r+') as manifest_file: try: - manifest = json.load(manifest_file) + manifest = json.loads(manifest_file.read()) return manifest except json.decoder.JSONDecodeError: log.warning(f'{source.name} contains malformed manifest ({source.parent_source.original_source})') @@ -2276,7 +2360,7 @@ def find_plugin_candidates(source: Union[LoadedSource, SourceDir], depth=2) -> l assert s.srctype == source.srctype, f'source dir {s.name}, {s.srctype} did not inherit {source.srctype} from {source.name}' assert s.parent_source == source.parent_source, f'source dir {s.name} did not inherit parent {source.parent_source} from {source.name}' - guess = InstInfo(source.name, source.location, source_dir=source) + guess = InstInfo(source.name, source.location, None, source_dir=source) guess.srctype = source.srctype manifest = None if guess.get_inst_details(): @@ -2324,8 +2408,7 @@ def available_plugins() -> list: clone = copy_remote_git_source(InstInfo(None, source.original_source, source_dir=source.content), - verbose=False, - parent_source=source) + verbose=False) clone.srctype = Source.GIT_LOCAL_CLONE clone.parent_source = source if not clone: From c6119e096b524a9e03999d4e6eee6a2c08385037 Mon Sep 17 00:00:00 2001 From: enaples Date: Thu, 12 Feb 2026 12:17:17 +0100 Subject: [PATCH 37/56] tests: makes `sed` works cross-platform --- tests/test_reckless.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_reckless.py b/tests/test_reckless.py index 71ef7a2a2be6..8e84cf162d56 100644 --- a/tests/test_reckless.py +++ b/tests/test_reckless.py @@ -62,7 +62,8 @@ def canned_github_server(directory): 'git add --all;' 'git commit -m "initial commit - autogenerated by test_reckless.py";') tag_and_update = ('git tag v1;' - "sed -i 's/v1/v2/g' testplugpass/testplugpass.py;" + "sed -i.bak 's/v1/v2/g' testplugpass/testplugpass.py;" + "rm -f testplugpass/testplugpass.py.bak;" 'git add testplugpass/testplugpass.py;' 'git commit -m "update to v2";' 'git tag v2;') From 564727f7a77f03d89d64a1ab10ee5882e89b84ac Mon Sep 17 00:00:00 2001 From: enaples Date: Thu, 12 Feb 2026 17:32:44 +0100 Subject: [PATCH 38/56] tests: added rejection for shebang `#!/usr/bin/env -S uv run --script` --- tools/reckless | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/reckless b/tools/reckless index 5182e7898f48..b188a782dabd 100755 --- a/tools/reckless +++ b/tools/reckless @@ -1234,7 +1234,7 @@ def check_for_shebang(source: InstInfo) -> bool: if entrypoint_file.split('\n')[0].startswith('#!'): # Calling the python interpreter will not manage dependencies. # Leave this to another python installer. - for interpreter in ['bin/python', 'env python']: + for interpreter in ['bin/python', 'env python', 'uv run']: if interpreter in entrypoint_file.split('\n')[0]: return False return True From 72572c4ea5605b31df0a64cd19111f2bfe776901 Mon Sep 17 00:00:00 2001 From: enaples Date: Thu, 12 Feb 2026 17:33:44 +0100 Subject: [PATCH 39/56] tests: test reckless installs a local plugin cloned from GH --- tests/test_reckless.py | 46 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/tests/test_reckless.py b/tests/test_reckless.py index 8e84cf162d56..832bb9c09a54 100644 --- a/tests/test_reckless.py +++ b/tests/test_reckless.py @@ -514,3 +514,49 @@ def test_reckless_usage(node_factory): with pytest.raises(lightning.RpcError, match="reckless: error: argument cmd1: invalid choice: 'saerch'"): node.rpc.reckless('saerch', 'testplugpass') + + +@pytest.mark.slow_test +@pytest.mark.flaky(max_runs=3) +@unittest.skipIf(VALGRIND, "virtual environment triggers memleak detection") +def test_reckless_install_from_real_py(node_factory): + """Test reckless install a real plugin""" + notification_plugin = os.path.join(os.getcwd(), 'tests/plugins/custom_notifications.py') + node = get_reckless_node(node_factory, options={"plugin": notification_plugin}) + + # Temporarily bypass `canned_github_server` fixture so reckless hits real Github repo + saved_redir = my_env.pop('REDIR_GITHUB', None) + + try: + r = reckless([f"--network={NETWORK}", "-v", "install", "backup"], + dir=node.lightning_dir) + assert r.returncode == 0 + assert r.search_stdout('plugin installed:') + + installed_path = (Path(node.lightning_dir) / 'reckless/backup').resolve() + assert installed_path.is_dir() + + network_dir = (Path(node.lightning_dir) / NETWORK).resolve() + backup_dest = str(network_dir / 'backup.bkp') + venv_python = str(installed_path / '.venv' / 'bin' / 'python') + backup_cli = str(installed_path / 'source' / 'backup' / 'backup-cli') + subprocess.run([venv_python, backup_cli, 'init', + '--lightning-dir', str(network_dir), + f'file://{backup_dest}'], + check=True, env=my_env, timeout=30) + + node.start() + + plugins = node.rpc.plugin_list()['plugins'] + plugin_names = [p['name'] for p in plugins] + assert any('backup' in name for name in plugin_names) + + r = reckless([f"--network={NETWORK}", "listavailable", "-v", "--json"], dir=node.lightning_dir) + assert r.returncode == 0 + assert r.search_stdout('backup') + + finally: + # Restore redirect for other tests + if saved_redir is not None: + my_env['REDIR_GITHUB'] = saved_redir + From 5bf2f77ab805946b8bdc0b2f7264e080e13b7572 Mon Sep 17 00:00:00 2001 From: enaples Date: Mon, 16 Feb 2026 16:01:24 +0100 Subject: [PATCH 40/56] tests: test real py plugin installation from commit --- tests/test_reckless.py | 47 ++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 45 insertions(+), 2 deletions(-) diff --git a/tests/test_reckless.py b/tests/test_reckless.py index 832bb9c09a54..59d8fdc9e2b5 100644 --- a/tests/test_reckless.py +++ b/tests/test_reckless.py @@ -519,8 +519,8 @@ def test_reckless_usage(node_factory): @pytest.mark.slow_test @pytest.mark.flaky(max_runs=3) @unittest.skipIf(VALGRIND, "virtual environment triggers memleak detection") -def test_reckless_install_from_real_py(node_factory): - """Test reckless install a real plugin""" +def test_reckless_install_py(node_factory): + """Test reckless install a real python plugin""" notification_plugin = os.path.join(os.getcwd(), 'tests/plugins/custom_notifications.py') node = get_reckless_node(node_factory, options={"plugin": notification_plugin}) @@ -560,3 +560,46 @@ def test_reckless_install_from_real_py(node_factory): if saved_redir is not None: my_env['REDIR_GITHUB'] = saved_redir +@pytest.mark.slow_test +@pytest.mark.flaky(max_runs=3) +@unittest.skipIf(VALGRIND, "virtual environment triggers memleak detection") +def test_reckless_install_from_commit_py(node_factory): + """Test reckless install a real python plugin from specifi commit""" + notification_plugin = os.path.join(os.getcwd(), 'tests/plugins/custom_notifications.py') + node = get_reckless_node(node_factory, options={"plugin": notification_plugin}) + + # Temporarily bypass `canned_github_server` fixture so reckless hits real Github repo + saved_redir = my_env.pop('REDIR_GITHUB', None) + from_commit = "478505fab37dd57a48834a5018016546729cf39a" + try: + r = reckless([f"--network={NETWORK}", "-v", "install", f"backup@{from_commit}"], + dir=node.lightning_dir) + assert r.returncode == 0 + assert r.search_stdout('plugin installed:') + + installed_path = (Path(node.lightning_dir) / 'reckless/backup').resolve() + assert installed_path.is_dir() + + network_dir = (Path(node.lightning_dir) / NETWORK).resolve() + backup_dest = str(network_dir / 'backup.bkp') + venv_python = str(installed_path / '.venv' / 'bin' / 'python') + backup_cli = str(installed_path / 'source' / 'backup' / 'backup-cli') + subprocess.run([venv_python, backup_cli, 'init', + '--lightning-dir', str(network_dir), + f'file://{backup_dest}'], + check=True, env=my_env, timeout=30) + + node.start() + + plugins = node.rpc.plugin_list()['plugins'] + plugin_names = [p['name'] for p in plugins] + assert any('backup' in name for name in plugin_names) + + r = reckless([f"--network={NETWORK}", "listavailable", "-v", "--json"], dir=node.lightning_dir) + assert r.returncode == 0 + assert r.search_stdout('backup') + + finally: + # Restore redirect for other tests + if saved_redir is not None: + my_env['REDIR_GITHUB'] = saved_redir \ No newline at end of file From bfcaee9eeda5b16d2f91126082d55a546c90e3bc Mon Sep 17 00:00:00 2001 From: enaples Date: Mon, 16 Feb 2026 16:52:59 +0100 Subject: [PATCH 41/56] tests: test reckless install plugin from github url --- tests/test_reckless.py | 42 +++++++++++++++++++++++++++++++++++++++++- 1 file changed, 41 insertions(+), 1 deletion(-) diff --git a/tests/test_reckless.py b/tests/test_reckless.py index 59d8fdc9e2b5..d8f264739183 100644 --- a/tests/test_reckless.py +++ b/tests/test_reckless.py @@ -519,7 +519,7 @@ def test_reckless_usage(node_factory): @pytest.mark.slow_test @pytest.mark.flaky(max_runs=3) @unittest.skipIf(VALGRIND, "virtual environment triggers memleak detection") -def test_reckless_install_py(node_factory): +def test_reckless_install_from_source_py(node_factory): """Test reckless install a real python plugin""" notification_plugin = os.path.join(os.getcwd(), 'tests/plugins/custom_notifications.py') node = get_reckless_node(node_factory, options={"plugin": notification_plugin}) @@ -560,6 +560,46 @@ def test_reckless_install_py(node_factory): if saved_redir is not None: my_env['REDIR_GITHUB'] = saved_redir +@pytest.mark.slow_test +@pytest.mark.flaky(max_runs=3) +@unittest.skipIf(VALGRIND, "virtual environment triggers memleak detection") +def test_reckless_install_from_github_url(node_factory): + """Test reckless installs a plugin given a full GitHub URL. + """ + # Bypass the canned local redirect so reckless clones from real GitHub. + saved_redir = my_env.pop('REDIR_GITHUB', None) + + notification_plugin = os.path.join(os.getcwd(), 'tests/plugins/custom_notifications.py') + node = get_reckless_node(node_factory, options={"plugin": notification_plugin}) + + github_url = "https://github.com/ca-ruz/bumpit" + plugin_name = "bumpit" + + try: + r = reckless([f"--network={NETWORK}", "-v", "install", github_url], + dir=node.lightning_dir, timeout=300) + assert r.returncode == 0 + assert r.search_stdout('plugin installed:') + + installed_path = (Path(node.lightning_dir) / 'reckless' / plugin_name).resolve() + assert installed_path.is_dir() + + node.start() + + plugins = node.rpc.plugin_list()['plugins'] + plugin_names = [p['name'] for p in plugins] + assert any(plugin_name in name for name in plugin_names) + + r = reckless([f"--network={NETWORK}", "listavailable", "-v", "--json"], + dir=node.lightning_dir) + assert r.returncode == 0 + assert r.search_stdout(plugin_name) + + finally: + if saved_redir is not None: + my_env['REDIR_GITHUB'] = saved_redir + + @pytest.mark.slow_test @pytest.mark.flaky(max_runs=3) @unittest.skipIf(VALGRIND, "virtual environment triggers memleak detection") From 3e634e62b8034384e6189ee904f4a22079e9d938 Mon Sep 17 00:00:00 2001 From: enaples Date: Tue, 17 Feb 2026 09:39:19 +0100 Subject: [PATCH 42/56] tests: test reckless ability to disable a plugin that caused a cln crash --- tests/test_reckless.py | 52 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/tests/test_reckless.py b/tests/test_reckless.py index d8f264739183..a3589a035bbf 100644 --- a/tests/test_reckless.py +++ b/tests/test_reckless.py @@ -600,6 +600,58 @@ def test_reckless_install_from_github_url(node_factory): my_env['REDIR_GITHUB'] = saved_redir +@pytest.mark.slow_test +@pytest.mark.flaky(max_runs=3) +@unittest.skipIf(VALGRIND, "virtual environment triggers memleak detection") +def test_install_crash_recovery(node_factory): + """Test reckless can uninstall a plugin that prevents CLN from starting. + + Installs the backup plugin via reckless, then replaces the entry + point with a script that exits immediately so CLN cannot start. + Verifies that reckless uninstall works while the node is stopped + and that CLN starts cleanly afterward. + """ + saved_redir = my_env.pop('REDIR_GITHUB', None) + + notification_plugin = os.path.join(os.getcwd(), 'tests/plugins/custom_notifications.py') + node = get_reckless_node(node_factory, + options={"plugin": notification_plugin}, + start=False) + node.may_fail = True + + try: + r = reckless([f"--network={NETWORK}", "-v", "install", "backup"], + dir=node.lightning_dir) + assert r.returncode == 0 + assert r.search_stdout('plugin installed:') + + installed_path = (Path(node.lightning_dir) / 'reckless' / 'backup').resolve() + assert installed_path.is_dir() + + node.daemon.start(wait_for_initialized=False) + rc = node.daemon.wait(timeout=60) + assert rc != 0, f"Expected CLN to crash but got exit code {rc}" + + r = reckless([f"--network={NETWORK}", "-v", "uninstall", "backup"], + dir=node.lightning_dir) + assert r.returncode == 0 + assert r.search_stdout('uninstalled') + + assert not installed_path.exists() + + config_path = Path(node.lightning_dir) / NETWORK / 'config' + if config_path.exists(): + config_text = config_path.read_text() + assert 'plugin=' not in config_text or 'backup' not in config_text + + node.may_fail = False + node.start() + + finally: + if saved_redir is not None: + my_env['REDIR_GITHUB'] = saved_redir + + @pytest.mark.slow_test @pytest.mark.flaky(max_runs=3) @unittest.skipIf(VALGRIND, "virtual environment triggers memleak detection") From 087c0a0f4f3a82df58a474f6360ceb5a0def57fe Mon Sep 17 00:00:00 2001 From: enaples Date: Tue, 17 Feb 2026 11:07:46 +0100 Subject: [PATCH 43/56] tests: making `test_install_crash_recovery` cln node to crash for real --- tests/test_reckless.py | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/tests/test_reckless.py b/tests/test_reckless.py index a3589a035bbf..37f7228883f5 100644 --- a/tests/test_reckless.py +++ b/tests/test_reckless.py @@ -606,10 +606,9 @@ def test_reckless_install_from_github_url(node_factory): def test_install_crash_recovery(node_factory): """Test reckless can uninstall a plugin that prevents CLN from starting. - Installs the backup plugin via reckless, then replaces the entry - point with a script that exits immediately so CLN cannot start. - Verifies that reckless uninstall works while the node is stopped - and that CLN starts cleanly afterward. + Installs the backup plugin via reckless RPC call, then verify that + CLN crashed. Use reckless tool to unintall the plugin that caused + that prevents CLN from starting and verify that CLN starts cleanly afterward. """ saved_redir = my_env.pop('REDIR_GITHUB', None) @@ -620,17 +619,17 @@ def test_install_crash_recovery(node_factory): node.may_fail = True try: - r = reckless([f"--network={NETWORK}", "-v", "install", "backup"], - dir=node.lightning_dir) - assert r.returncode == 0 - assert r.search_stdout('plugin installed:') + node.start() + with pytest.raises(lightning.RpcError): + node.rpc.reckless('install', 'backup') + rc = node.daemon.wait(timeout=10) + assert rc != 0, f"Expected CLN to crash but got exit code {rc}" installed_path = (Path(node.lightning_dir) / 'reckless' / 'backup').resolve() assert installed_path.is_dir() - node.daemon.start(wait_for_initialized=False) - rc = node.daemon.wait(timeout=60) - assert rc != 0, f"Expected CLN to crash but got exit code {rc}" + installed_path = (Path(node.lightning_dir) / 'reckless' / 'backup').resolve() + assert installed_path.is_dir() r = reckless([f"--network={NETWORK}", "-v", "uninstall", "backup"], dir=node.lightning_dir) From 0abde9ec6b3403dc7fde683876a9b848b0e73743 Mon Sep 17 00:00:00 2001 From: enaples Date: Tue, 17 Feb 2026 11:11:23 +0100 Subject: [PATCH 44/56] tests: test reckless installs a plugin from a specific commit via RPC --- tests/test_reckless.py | 43 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/tests/test_reckless.py b/tests/test_reckless.py index 37f7228883f5..5c2d3589e1c6 100644 --- a/tests/test_reckless.py +++ b/tests/test_reckless.py @@ -692,5 +692,48 @@ def test_reckless_install_from_commit_py(node_factory): finally: # Restore redirect for other tests + if saved_redir is not None: + my_env['REDIR_GITHUB'] = saved_redir + + +@pytest.mark.slow_test +@pytest.mark.flaky(max_runs=3) +@unittest.skipIf(VALGRIND, "virtual environment triggers memleak detection") +def test_reckless_rpc_install_from_commit_py(node_factory): + """Test reckless installs a plugin at a specific commit via RPC.""" + saved_redir = my_env.pop('REDIR_GITHUB', None) + + notification_plugin = os.path.join(os.getcwd(), 'tests/plugins/custom_notifications.py') + node = get_reckless_node(node_factory, options={"plugin": notification_plugin}) + + plugin_name = "currencyrate" + from_commit = "306cfadcc90e98ce6fb2e27915efa72be0f66ad6" + + try: + node.start() + + result = node.rpc.reckless('install', f'{plugin_name}@{from_commit}') + assert 'plugin installed:' in ''.join(result.get('log', [])) + + installed_path = (Path(node.lightning_dir) / 'reckless' / plugin_name).resolve() + assert installed_path.is_dir() + + # Verify the .metadata file records the correct commit. + metadata_file = installed_path / '.metadata' + assert metadata_file.exists(), ".metadata file not found" + metadata_text = metadata_file.read_text() + metadata_lines = metadata_text.splitlines() + metadata = {} + for i, line in enumerate(metadata_lines): + if i > 0: + metadata[metadata_lines[i - 1].strip()] = line.strip() + + assert metadata.get('requested commit') == from_commit, \ + f"requested commit mismatch: {metadata.get('requested commit')}" + assert metadata.get('installed commit') is not None + assert metadata['installed commit'].startswith(from_commit[:7]), \ + f"installed commit {metadata['installed commit']} does not match {from_commit}" + + finally: if saved_redir is not None: my_env['REDIR_GITHUB'] = saved_redir \ No newline at end of file From b11b06376188c672f704152a10d90a852334491b Mon Sep 17 00:00:00 2001 From: enaples Date: Tue, 17 Feb 2026 11:30:30 +0100 Subject: [PATCH 45/56] tests: test reckless uninstall cleans confing from just-installed plugin --- tests/test_reckless.py | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/tests/test_reckless.py b/tests/test_reckless.py index 5c2d3589e1c6..b2ba55ca231f 100644 --- a/tests/test_reckless.py +++ b/tests/test_reckless.py @@ -734,6 +734,41 @@ def test_reckless_rpc_install_from_commit_py(node_factory): assert metadata['installed commit'].startswith(from_commit[:7]), \ f"installed commit {metadata['installed commit']} does not match {from_commit}" + finally: + if saved_redir is not None: + my_env['REDIR_GITHUB'] = saved_redir + + +@pytest.mark.slow_test +@pytest.mark.flaky(max_runs=3) +@unittest.skipIf(VALGRIND, "virtual environment triggers memleak detection") +def test_reckless_rpc_unistall_cleans_config(node_factory): + """Test reckless uninstall cleans confing from just-installed plugin.""" + saved_redir = my_env.pop('REDIR_GITHUB', None) + + notification_plugin = os.path.join(os.getcwd(), 'tests/plugins/custom_notifications.py') + node = get_reckless_node(node_factory, options={"plugin": notification_plugin}) + + plugin_name = "currencyrate" + + try: + node.start() + + result = node.rpc.reckless('install', plugin_name) + assert 'plugin installed:' in ''.join(result.get('log', [])) + + installed_path = (Path(node.lightning_dir) / 'reckless' / plugin_name).resolve() + assert installed_path.is_dir() + + result = node.rpc.reckless('uninstall', plugin_name) + assert f'{plugin_name} uninstalled successfully.' in ''.join(result.get('log', [])) + assert not installed_path.is_dir() + + config_path = Path(node.lightning_dir) / NETWORK / 'config' + if config_path.exists(): + config_text = config_path.read_text() + assert 'disable-plugin=' not in config_text or plugin_name not in config_text + finally: if saved_redir is not None: my_env['REDIR_GITHUB'] = saved_redir \ No newline at end of file From ee3e168d134f507ee51c4e1a206f8b03110d42ac Mon Sep 17 00:00:00 2001 From: enaples Date: Wed, 18 Feb 2026 10:18:48 +0100 Subject: [PATCH 46/56] tests: test reckless returns well-formatted json --- tests/test_reckless.py | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/tests/test_reckless.py b/tests/test_reckless.py index b2ba55ca231f..11a5a3e5b735 100644 --- a/tests/test_reckless.py +++ b/tests/test_reckless.py @@ -479,6 +479,37 @@ def test_reckless_available(node_factory): assert r.search_stdout('testpluguv') +def test_listavailable_json_format(node_factory): + """Verify listavailable --json returns properly structured JSON objects.""" + n = get_reckless_node(node_factory) + r = reckless([f"--network={NETWORK}", "listavailable", "--json"], + dir=n.lightning_dir) + assert r.returncode == 0 + + result = json.loads(''.join(r.stdout)) + plugins = result['result'] + assert isinstance(plugins, list) + assert len(plugins) > 0 + + expected_fields = {'name', 'short_description', 'long_description', + 'entrypoint', 'requirements'} + + for plugin in plugins: + + assert isinstance(plugin, dict), \ + f"expected dict but got {type(plugin).__name__}: {plugin!r}" + + missing = expected_fields - plugin.keys() + assert not missing, \ + f"plugin {plugin.get('name', '?')} missing fields: {missing}" + + assert isinstance(plugin['name'], str) and plugin['name'] + assert isinstance(plugin['entrypoint'], str) and plugin['entrypoint'] + + assert isinstance(plugin['requirements'], list), \ + f"plugin {plugin['name']}: requirements should be a list" + + @pytest.mark.timeout(120) def test_reckless_notifications(node_factory): """Reckless streams logs to the reckless-rpc plugin which are emitted From b335174cea19fdba0ae79df168f8d9912a6eae37 Mon Sep 17 00:00:00 2001 From: enaples Date: Wed, 18 Feb 2026 14:06:24 +0100 Subject: [PATCH 47/56] reckless: latest version --- tools/reckless | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/reckless b/tools/reckless index b188a782dabd..9cef41f9dbec 100755 --- a/tools/reckless +++ b/tools/reckless @@ -1234,7 +1234,7 @@ def check_for_shebang(source: InstInfo) -> bool: if entrypoint_file.split('\n')[0].startswith('#!'): # Calling the python interpreter will not manage dependencies. # Leave this to another python installer. - for interpreter in ['bin/python', 'env python', 'uv run']: + for interpreter in ['bin/python', 'env python']: if interpreter in entrypoint_file.split('\n')[0]: return False return True @@ -2686,4 +2686,4 @@ if __name__ == '__main__': log.reply_json() # We're done streaming to this socket, but the rpc plugin will close it. if log.socket: - log.socket.shutdown(socket.SHUT_WR) + log.socket.shutdown(socket.SHUT_WR) \ No newline at end of file From 0815ff6a6d58d8b06b0f3f4d6af3587998506c3f Mon Sep 17 00:00:00 2001 From: enaples Date: Wed, 18 Feb 2026 14:25:17 +0100 Subject: [PATCH 48/56] reckless: version before wrong rebasing --- tools/reckless | 101 +++++-------------------------------------------- 1 file changed, 9 insertions(+), 92 deletions(-) diff --git a/tools/reckless b/tools/reckless index 9cef41f9dbec..78412f4d4877 100755 --- a/tools/reckless +++ b/tools/reckless @@ -385,8 +385,9 @@ class SourceDir(): self.contents = populate_local_dir(self.location) elif self.srctype in [Source.LOCAL_REPO, Source.GIT_LOCAL_CLONE]: self.contents = populate_local_repo(self.location, parent=self, parent_source=self.parent_source) - elif self.srctype == Source.GITHUB_REPO: - self.contents = populate_github_repo(self.location) + elif self.srctype == Source.REMOTE_GIT_REPO: + self.contents = copy_remote_git_source(InstInfo(self.name, self.location)).contents + else: raise Exception("populate method undefined for {self.srctype}") # Ensure the relative path of the contents is inherited. @@ -659,92 +660,7 @@ def populate_local_repo(path: str, parent=None, parent_source=None) -> list: return basedir.contents -def source_element_from_repo_api(member: dict): - # api accessed via /contents/ - if 'type' in member and 'name' in member and 'git_url' in member: - if member['type'] == 'dir': - return SourceDir(member['git_url'], srctype=Source.GITHUB_REPO, - name=member['name']) - elif member['type'] == 'file': - # Likely a submodule - if member['size'] == 0: - return SourceDir(None, srctype=Source.GITHUB_REPO, - name=member['name']) - return SourceFile(member['name']) - elif member['type'] == 'commit': - # No path is given by the api here - return SourceDir(None, srctype=Source.GITHUB_REPO, - name=member['name']) - # git_url with /tree/ presents results a little differently - elif 'type' in member and 'path' in member and 'url' in member: - if member['type'] not in ['tree', 'blob']: - log.debug(f' skipping {member["path"]} type={member["type"]}') - if member['type'] == 'tree': - return SourceDir(member['url'], srctype=Source.GITHUB_REPO, - name=member['path']) - elif member['type'] == 'blob': - # This can be a submodule - if member['size'] == 0: - return SourceDir(member['git_url'], srctype=Source.GITHUB_REPO, - name=member['name']) - return SourceFile(member['path']) - elif member['type'] == 'commit': - # No path is given by the api here - return SourceDir(None, srctype=Source.GITHUB_REPO, - name=member['name']) - return None - - -def populate_github_repo(url: str) -> list: - """populate one level of a github repository via REST API""" - # Forces search to clone remote repos (for blackbox testing) - if GITHUB_API_FALLBACK: - with tempfile.NamedTemporaryFile() as tmp: - raise HTTPError(url, 403, 'simulated ratelimit', {}, tmp) - # FIXME: This probably contains leftover cruft. - repo = url.split('/') - while '' in repo: - repo.remove('') - repo_name = None - parsed_url = urlparse(url.removesuffix('.git')) - if 'github.com' not in parsed_url.netloc: - return None - if len(parsed_url.path.split('/')) < 2: - return None - start = 1 - # Maybe we were passed an api.github.com/repo/ url - if 'api' in parsed_url.netloc: - start += 1 - repo_user = parsed_url.path.split('/')[start] - repo_name = parsed_url.path.split('/')[start + 1] - - # Get details from the github API. - if API_GITHUB_COM in url: - api_url = url - else: - api_url = f'{API_GITHUB_COM}/repos/{repo_user}/{repo_name}/contents/' - - git_url = api_url - if "api.github.com" in git_url: - # This lets us redirect to handle blackbox testing - log.debug(f'fetching from gh API: {git_url}') - git_url = (API_GITHUB_COM + git_url.split("api.github.com")[-1]) - # Ratelimiting occurs for non-authenticated GH API calls at 60 in 1 hour. - r = urlopen(git_url, timeout=5) - if r.status != 200: - return False - if 'git/tree' in git_url: - tree = json.loads(r.read().decode())['tree'] - else: - tree = json.loads(r.read().decode()) - contents = [] - for sub in tree: - if source_element_from_repo_api(sub): - contents.append(source_element_from_repo_api(sub)) - return contents - - -def copy_remote_git_source(github_source: InstInfo, verbose: bool=True) -> SourceDir: +def copy_remote_git_source(github_source: InstInfo, verbose: bool=True, parent_source=None) -> SourceDir: """clone or fetch & checkout a local copy of a remote git repo""" user, repo = Source.get_github_user_repo(github_source.source_loc) if not user or not repo: @@ -2336,7 +2252,7 @@ def fetch_manifest(source: SourceDir) -> dict: return None with open(path, 'r+') as manifest_file: try: - manifest = json.loads(manifest_file.read()) + manifest = json.load(manifest_file) return manifest except json.decoder.JSONDecodeError: log.warning(f'{source.name} contains malformed manifest ({source.parent_source.original_source})') @@ -2360,7 +2276,7 @@ def find_plugin_candidates(source: Union[LoadedSource, SourceDir], depth=2) -> l assert s.srctype == source.srctype, f'source dir {s.name}, {s.srctype} did not inherit {source.srctype} from {source.name}' assert s.parent_source == source.parent_source, f'source dir {s.name} did not inherit parent {source.parent_source} from {source.name}' - guess = InstInfo(source.name, source.location, None, source_dir=source) + guess = InstInfo(source.name, source.location, source_dir=source) guess.srctype = source.srctype manifest = None if guess.get_inst_details(): @@ -2408,7 +2324,8 @@ def available_plugins() -> list: clone = copy_remote_git_source(InstInfo(None, source.original_source, source_dir=source.content), - verbose=False) + verbose=False, + parent_source=source) clone.srctype = Source.GIT_LOCAL_CLONE clone.parent_source = source if not clone: @@ -2686,4 +2603,4 @@ if __name__ == '__main__': log.reply_json() # We're done streaming to this socket, but the rpc plugin will close it. if log.socket: - log.socket.shutdown(socket.SHUT_WR) \ No newline at end of file + log.socket.shutdown(socket.SHUT_WR) From 258faaca1aa592ebd0adbe8d26ae125fd8170fad Mon Sep 17 00:00:00 2001 From: enaples Date: Wed, 18 Feb 2026 16:15:48 +0100 Subject: [PATCH 49/56] reckless: resolve symlink `/tmp/ to `/private/tmp` --- tests/test_reckless.py | 2 +- tools/reckless | 13 +++++++------ 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/tests/test_reckless.py b/tests/test_reckless.py index 11a5a3e5b735..baf1470f745d 100644 --- a/tests/test_reckless.py +++ b/tests/test_reckless.py @@ -387,7 +387,7 @@ def test_disable_enable(node_factory): dir=n.lightning_dir) assert r.returncode == 0 assert r.search_stdout('testplugpass enabled') - test_plugin = {'name': str(plugin_path / 'testplugpass.py'), + test_plugin = {'name': str((plugin_path / 'testplugpass.py').resolve()), 'active': True, 'dynamic': True} time.sleep(1) print(n.rpc.plugin_list()['plugins']) diff --git a/tools/reckless b/tools/reckless index 78412f4d4877..4c91d97bac56 100755 --- a/tools/reckless +++ b/tools/reckless @@ -935,16 +935,17 @@ def create_wrapper(plugin: InstInfo): then run the plugin from within the same process.''' assert hasattr(plugin, 'venv') venv_full_path = Path(plugin.source_loc) / plugin.venv + real_source_loc = str(Path(plugin.source_loc).resolve()) with open(Path(plugin.source_loc) / plugin.entry, 'w') as wrapper: wrapper.write((f"#!{venv_full_path}/bin/python\n" "import sys\n" "import runpy\n\n" - f"if '{plugin.source_loc}/{plugin.subdir}' not in " + f"if '{real_source_loc}/{plugin.subdir}' not in " "sys.path:\n" - f" sys.path.append('{plugin.source_loc}/" + f" sys.path.append('{real_source_loc}/" f"{plugin.subdir}')\n" - f"if '{plugin.source_loc}' in sys.path:\n" - f" sys.path.remove('{plugin.source_loc}')\n" + f"if '{real_source_loc}' in sys.path:\n" + f" sys.path.remove('{real_source_loc}')\n" f"runpy.run_module(\"{plugin.name}\", " "{}, \"__main__\")")) wrapper_file = Path(plugin.source_loc) / plugin.entry @@ -1875,7 +1876,7 @@ def enable(plugin_name: str): except NotFoundError as err: log.error(err) return None - path = inst.entry + path = str(Path(inst.entry).resolve()) if not Path(path).exists(): log.error(f'cannot find installed plugin at expected path {path}') return None @@ -1907,7 +1908,7 @@ def disable(plugin_name: str): except NotFoundError as err: log.warning(f'failed to disable: {err}') return None - path = inst.entry + path = str(Path(inst.entry).resolve()) if not Path(path).exists(): sys.stderr.write(f'Could not find plugin at {path}\n') return None From ddaa4fa901df4cdea10e58263e6250b9abb2de3e Mon Sep 17 00:00:00 2001 From: enaples Date: Wed, 18 Feb 2026 17:07:15 +0100 Subject: [PATCH 50/56] reckless: add `uv run` shebang --- tools/reckless | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/reckless b/tools/reckless index 4c91d97bac56..562408048611 100755 --- a/tools/reckless +++ b/tools/reckless @@ -1151,7 +1151,7 @@ def check_for_shebang(source: InstInfo) -> bool: if entrypoint_file.split('\n')[0].startswith('#!'): # Calling the python interpreter will not manage dependencies. # Leave this to another python installer. - for interpreter in ['bin/python', 'env python']: + for interpreter in ['bin/python', 'env python', 'uv run']: if interpreter in entrypoint_file.split('\n')[0]: return False return True From a83bc13a05f9b2f040bbaf121e81564ba239cbe7 Mon Sep 17 00:00:00 2001 From: enaples Date: Wed, 18 Feb 2026 17:13:26 +0100 Subject: [PATCH 51/56] tests: fix `test_shebang_install` The change allows `uv run --script` shebangs through to the shebang installer, while still rejecting plain `bin/python`, `env python`, and project-mode uv run (without --script) shebangs that need a separate installer to manage dependencies. --- tools/reckless | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/tools/reckless b/tools/reckless index 562408048611..637513bf1203 100755 --- a/tools/reckless +++ b/tools/reckless @@ -1151,9 +1151,14 @@ def check_for_shebang(source: InstInfo) -> bool: if entrypoint_file.split('\n')[0].startswith('#!'): # Calling the python interpreter will not manage dependencies. # Leave this to another python installer. - for interpreter in ['bin/python', 'env python', 'uv run']: - if interpreter in entrypoint_file.split('\n')[0]: + # Exception: 'uv run --script' uses PEP 723 inline metadata and + # is self-contained — let the shebang installer handle it. + shebang_line = entrypoint_file.split('\n')[0] + for interpreter in ['bin/python', 'env python']: + if interpreter in shebang_line: return False + if 'uv run' in shebang_line and '--script' not in shebang_line: + return False return True return False From c14ff6cb5b51bd76dd2e5d8a8a715490e2e17571 Mon Sep 17 00:00:00 2001 From: enaples Date: Wed, 18 Feb 2026 17:34:04 +0100 Subject: [PATCH 52/56] tests: fixing test with real plugins --- tests/test_reckless.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tests/test_reckless.py b/tests/test_reckless.py index baf1470f745d..bffc4661adc4 100644 --- a/tests/test_reckless.py +++ b/tests/test_reckless.py @@ -569,9 +569,8 @@ def test_reckless_install_from_source_py(node_factory): network_dir = (Path(node.lightning_dir) / NETWORK).resolve() backup_dest = str(network_dir / 'backup.bkp') - venv_python = str(installed_path / '.venv' / 'bin' / 'python') backup_cli = str(installed_path / 'source' / 'backup' / 'backup-cli') - subprocess.run([venv_python, backup_cli, 'init', + subprocess.run([backup_cli, 'init', '--lightning-dir', str(network_dir), f'file://{backup_dest}'], check=True, env=my_env, timeout=30) @@ -704,9 +703,8 @@ def test_reckless_install_from_commit_py(node_factory): network_dir = (Path(node.lightning_dir) / NETWORK).resolve() backup_dest = str(network_dir / 'backup.bkp') - venv_python = str(installed_path / '.venv' / 'bin' / 'python') backup_cli = str(installed_path / 'source' / 'backup' / 'backup-cli') - subprocess.run([venv_python, backup_cli, 'init', + subprocess.run([backup_cli, 'init', '--lightning-dir', str(network_dir), f'file://{backup_dest}'], check=True, env=my_env, timeout=30) From 029ae2a4e2c9227e163320c57dea21da755f1966 Mon Sep 17 00:00:00 2001 From: ScuttoZ Date: Tue, 21 Apr 2026 16:20:31 +0200 Subject: [PATCH 53/56] reckless: A plugin with required option now doesn't crash installation the first time reckless tries to install it without the option, but instead warns the user and asks for manual insertion, then resumes installation attempt. --- tools/reckless | 43 +++++++++++++++++++++++++++++++++++++++---- 1 file changed, 39 insertions(+), 4 deletions(-) diff --git a/tools/reckless b/tools/reckless index 637513bf1203..9277976071de 100755 --- a/tools/reckless +++ b/tools/reckless @@ -1588,10 +1588,45 @@ def _install_plugin(src: InstInfo) -> Union[InstInfo, None]: log.debug("plugin testing error:") for line in test_log: log.debug(f' {line}') - log.error('plugin testing failed') - remove_dir(clone_path) - remove_dir(inst_path) - return None + log.warning('plugin exited before establishing a CLN connection; ' + 'it may require options to be configured') + # At this point the installation cannot continue without user input. + # This part might be incompatible with automated executions (e.g. CI) + user_input = '' + if sys.stdin.isatty(): + print("Enter an option as NAME=VALUE to retry (string values only), " + "or press Enter to skip: ", end='', flush=True) + try: + user_input = input().strip() + except EOFError: + pass + if user_input: + if '=' not in user_input: + log.warning("invalid option format (expected NAME=VALUE); skipping retry") + else: + opt_name, opt_value = user_input.split('=', 1) + opt_name = opt_name.strip() + if not opt_name: + log.warning("option name cannot be empty; skipping retry") + elif not isinstance(opt_value, str): + log.warning("option value must be a string; skipping retry") + else: + retry_env = os.environ.copy() + retry_env[opt_name] = opt_value + log.debug(f"retrying plugin test with {opt_name}={opt_value!r}") + try: + retest = run( + [Path(staged_src.source_loc).joinpath(staged_src.entry)], + cwd=str(staging_path), stdout=PIPE, stderr=PIPE, + text=True, timeout=10, env=retry_env) + retest_returncode = retest.returncode + except TimeoutExpired: + retest_returncode = 0 + if retest_returncode == 0: + log.info('plugin started successfully with provided option') + else: + log.warning('plugin still failed after retry; ' + 'options must be configured before use') add_installation_metadata(staged_src, src) log.info(f'plugin installed: {inst_path}') From 23ef15c2e4a90704e274cd958aadd3b4377dcf6c Mon Sep 17 00:00:00 2001 From: ScuttoZ Date: Tue, 21 Apr 2026 16:22:47 +0200 Subject: [PATCH 54/56] reckless: Added test plugin with required options (added to canned server too) --- .../testplugreqopts/requirements.txt | 1 + .../testplugreqopts/testplugreqopts.py | 22 +++++++++ .../rkls_api_lightningd_plugins.json | 47 +++++++++++++++++++ 3 files changed, 70 insertions(+) create mode 100644 tests/data/recklessrepo/lightningd/testplugreqopts/requirements.txt create mode 100755 tests/data/recklessrepo/lightningd/testplugreqopts/testplugreqopts.py create mode 100644 tests/data/recklessrepo/rkls_api_lightningd_plugins.json diff --git a/tests/data/recklessrepo/lightningd/testplugreqopts/requirements.txt b/tests/data/recklessrepo/lightningd/testplugreqopts/requirements.txt new file mode 100644 index 000000000000..0096004b4a8d --- /dev/null +++ b/tests/data/recklessrepo/lightningd/testplugreqopts/requirements.txt @@ -0,0 +1 @@ +pyln-client diff --git a/tests/data/recklessrepo/lightningd/testplugreqopts/testplugreqopts.py b/tests/data/recklessrepo/lightningd/testplugreqopts/testplugreqopts.py new file mode 100755 index 000000000000..7d29417ebdfd --- /dev/null +++ b/tests/data/recklessrepo/lightningd/testplugreqopts/testplugreqopts.py @@ -0,0 +1,22 @@ +#!/usr/bin/env python3 +"""Plugin that requires a mandatory option. Exits immediately when the +option is absent - as happens when reckless runs the plugin standalone +outside of a CLN connection.""" +import os +import sys + +if not os.environ.get('TESTPLUG_REQUIRED_OPT'): + print("required option 'required-opt' is not configured", file=sys.stderr) + sys.exit(1) + +from pyln.client import Plugin + +plugin = Plugin() + + +@plugin.init() +def init(options, configuration, plugin, **kwargs): + plugin.log("testplugreqopts initialized") + + +plugin.run() diff --git a/tests/data/recklessrepo/rkls_api_lightningd_plugins.json b/tests/data/recklessrepo/rkls_api_lightningd_plugins.json new file mode 100644 index 000000000000..5d6339abe742 --- /dev/null +++ b/tests/data/recklessrepo/rkls_api_lightningd_plugins.json @@ -0,0 +1,47 @@ +[ + { + "name": "testplugpass", + "path": "testplugpass", + "url": "https://api.github.com/repos/lightningd/plugins/contents/webhook?ref=master", + "html_url": "https://github.com/lightningd/plugins/tree/master/testplugpass", + "git_url": "https://api.github.com/repos/lightningd/plugins/git/trees/testplugpass", + "download_url": null, + "type": "dir" + }, + { + "name": "testpluguv", + "path": "testpluguv", + "url": "https://api.github.com/repos/lightningd/plugins/contents/webhook?ref=master", + "html_url": "https://github.com/lightningd/plugins/tree/master/testpluguv", + "git_url": "https://api.github.com/repos/lightningd/plugins/git/trees/testpluguv", + "download_url": null, + "type": "dir" + }, + { + "name": "testplugfail", + "path": "testplugfail", + "url": "https://api.github.com/repos/lightningd/plugins/contents/testplugfail?ref=master", + "html_url": "https://github.com/lightningd/plugins/tree/master/testplugfail", + "git_url": "https://api.github.com/repos/lightningd/plugins/git/trees/testplugfail", + "download_url": null, + "type": "dir" + }, + { + "name": "testplugpyproj", + "path": "testplugpyproj", + "url": "https://api.github.com/repos/lightningd/plugins/contents/webhook?ref=master", + "html_url": "https://github.com/lightningd/plugins/tree/master/testplugpyproj", + "git_url": "https://api.github.com/repos/lightningd/plugins/git/trees/testplugpyproj", + "download_url": null, + "type": "dir" + }, + { + "name": "testplugreqopts", + "path": "testplugreqopts", + "url": "https://api.github.com/repos/lightningd/plugins/contents/testplugreqopts?ref=master", + "html_url": "https://github.com/lightningd/plugins/tree/master/testplugreqopts", + "git_url": "https://api.github.com/repos/lightningd/plugins/git/trees/testplugreqopts", + "download_url": null, + "type": "dir" + } +] From 99f02b3a5b4a4d6873793fd10bbbc4e030e1e270 Mon Sep 17 00:00:00 2001 From: ScuttoZ Date: Tue, 21 Apr 2026 16:23:14 +0200 Subject: [PATCH 55/56] test_reckless: Added test for plugins with required options --- tests/test_reckless.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/tests/test_reckless.py b/tests/test_reckless.py index bffc4661adc4..139660a4c62a 100644 --- a/tests/test_reckless.py +++ b/tests/test_reckless.py @@ -432,6 +432,20 @@ def test_tag_install(node_factory): header = line +def test_install_plugin_requiring_opts(node_factory): + """A plugin that exits non-zero when run standalone (e.g. because a + required option is not yet configured) should still install successfully.""" + n = get_reckless_node(node_factory) + r = reckless([f"--network={NETWORK}", "-v", "install", "testplugreqopts"], + dir=n.lightning_dir) + assert r.returncode == 0 + assert r.search_stdout('plugin installed:') + assert r.search_stdout('testplugreqopts enabled') + assert r.search_stdout('may require options') + plugin_path = Path(n.lightning_dir) / 'reckless/testplugreqopts' + assert plugin_path.exists() + + # Note: uv timeouts from the GH network seem to happen? @pytest.mark.slow_test @unittest.skipIf(VALGRIND, "node too slow for starting plugin under valgrind") From 1a9fe6649a493be5d0a204e4adaa555ae2ab1f48 Mon Sep 17 00:00:00 2001 From: ScuttoZ Date: Wed, 22 Apr 2026 14:56:02 +0200 Subject: [PATCH 56/56] ran flake8 --- tests/test_reckless.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_reckless.py b/tests/test_reckless.py index 139660a4c62a..12434342ba32 100644 --- a/tests/test_reckless.py +++ b/tests/test_reckless.py @@ -604,6 +604,7 @@ def test_reckless_install_from_source_py(node_factory): if saved_redir is not None: my_env['REDIR_GITHUB'] = saved_redir + @pytest.mark.slow_test @pytest.mark.flaky(max_runs=3) @unittest.skipIf(VALGRIND, "virtual environment triggers memleak detection") @@ -814,4 +815,4 @@ def test_reckless_rpc_unistall_cleans_config(node_factory): finally: if saved_redir is not None: - my_env['REDIR_GITHUB'] = saved_redir \ No newline at end of file + my_env['REDIR_GITHUB'] = saved_redir